Disable health tracing and endpoints during debug

Author: keithralphs

helm/blueapi/README.md

@@ -44,7 +44,7 @@ A Helm chart deploying a worker pod that runs Bluesky plans
 | serviceAccount.name | string | `""` |  |
 | startupProbe | object | `{"failureThreshold":5,"httpGet":{"path":"/healthz","port":"http"},"periodSeconds":10}` | A more lenient livenessProbe to allow the service to start fully. This is automatically disabled when in debug mode. |
 | tolerations | list | `[]` | May be required to run on specific nodes (e.g. the control machine) |
-| tracing | object | `{"otlp":{"enabled":false,"protocol":"http/protobuf","server":{"host":"http://opentelemetry-collector.tracing","port":4318}}}` | Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters |
+| tracing | object | `{"fastapi":{"excludedURLs":"/healthz"},"otlp":{"enabled":false,"protocol":"http/protobuf","server":{"host":"http://opentelemetry-collector.tracing","port":4318}}}` | Exclude health probe requests from tracing by default to prevent spamming |
 | volumeMounts | list | `[{"mountPath":"/config","name":"worker-config","readOnly":true}]` | Additional volumeMounts on the output StatefulSet definition. Define how volumes are mounted to the container referenced by using the same name. |
 | volumes | list | `[]` | Additional volumes on the output StatefulSet definition. Define volumes from e.g. Secrets, ConfigMaps or the Filesystem |
 | worker | object | `{"api":{"url":"http://0.0.0.0:8000/"},"env":{"sources":[{"kind":"planFunctions","module":"dodal.plans"},{"kind":"planFunctions","module":"dodal.plan_stubs.wrapped"}]},"logging":{"graylog":{"enabled":false,"url":"tcp://graylog-log-target.diamond.ac.uk:12231/"},"level":"INFO"},"scratch":{"repositories":[],"root":"/workspace"},"stomp":{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}}` | Config for the worker goes here, will be mounted into a config file |

helm/blueapi/templates/configmap.yaml

@@ -19,6 +19,8 @@ data:
   OTEL_EXPORTER_OTLP_ENDPOINT: {{ required "OTLP export enabled but server address not set" .Values.tracing.otlp.server.host }}:{{ .Values.tracing.otlp.server.port | default 4318 }}
 {{ end }}
 
+  OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: {{ .Values.tracing.fastapi.excludedURLs | default "" }}
+
 ---
 
 {{- if .Values.initContainer.enabled }}

helm/blueapi/templates/statefulset.yaml

@@ -182,7 +182,7 @@ spec:
           - "true"
           - "-m"
           - "blueapi"
-          {{- end }}
+          {{- else }}
           {{- with .Values.livenessProbe }}
           livenessProbe:
             {{- toYaml . | nindent 12 }}
@@ -195,12 +195,17 @@ spec:
           startupProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- end }}          
           envFrom:
             - configMapRef:
                 name: {{ include "blueapi.fullname" . }}-otel-config
           env:
             - name: UV_CACHE_DIR
               value: {{ (.Values.worker.scratch).root }}/.uv-cache
+            {{- if .Values.debug.enabled }}
+            - name: DEBUG_MODE
+              value: ON           
+            {{- end }}
             {{- if .Values.extraEnvVars }}
               {{- toYaml .Values.extraEnvVars | nindent 12 }}
             {{- end }}

helm/blueapi/values.schema.json

@@ -296,6 +296,14 @@
             "description": "Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters",
             "type": "object",
             "properties": {
+                "fastapi": {
+                    "type": "object",
+                    "properties": {
+                        "excludedURLs": {
+                            "type": "string"
+                        }
+                    }
+                },
                 "otlp": {
                     "type": "object",
                     "properties": {

helm/blueapi/values.yaml

@@ -163,13 +163,16 @@ extraEnvVars: []
 #       key: tiled_api_key
 
 # -- Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters
+# -- Exclude health probe requests from tracing by default to prevent spamming
 tracing:
   otlp:
     enabled: false
     protocol: http/protobuf
     server:
       host: http://opentelemetry-collector.tracing
       port: 4318
+  fastapi:
+    excludedURLs: "/healthz"
 
 # @schema $ref: config_schema.json
 # -- Config for the worker goes here, will be mounted into a config file