From e504979f66c1a6deeddb1c065b7915bbad6b0588 Mon Sep 17 00:00:00 2001 From: shekhargit1912 Date: Wed, 15 Jul 2026 22:40:12 +0530 Subject: [PATCH 1/2] feat(helm): expose OTEL headers, service name, and resource attributes The controller already honors the standard OpenTelemetry SDK env vars, but the chart had no way to set them, leaving authenticated OTLP backends (Langfuse Cloud, Honeycomb, Datadog, Grafana Cloud, New Relic) unreachable without post-render hacks. Add optional values, all rendered into the controller ConfigMap only when set so existing installs render identically: - otel.tracing.exporter.otlp.headers -> OTEL_EXPORTER_OTLP_TRACES_HEADERS - otel.logging.exporter.otlp.headers -> OTEL_EXPORTER_OTLP_LOGS_HEADERS - otel.serviceName -> OTEL_SERVICE_NAME - otel.resourceAttributes -> OTEL_RESOURCE_ATTRIBUTES Header and attribute maps render as comma-joined key=value pairs with sorted keys, matching the DEFAULT_AGENT_POD_LABELS pattern. Secret header values should instead be injected via controller.env/envFrom, as documented in values.yaml. Fixes #2126 Signed-off-by: shekhargit1912 --- .../templates/controller-configmap.yaml | 24 ++++++++ .../tests/controller-deployment_test.yaml | 57 +++++++++++++++++++ helm/kagent/values.yaml | 17 ++++++ 3 files changed, 98 insertions(+) diff --git a/helm/kagent/templates/controller-configmap.yaml b/helm/kagent/templates/controller-configmap.yaml index a3b519b6a..8242129b3 100644 --- a/helm/kagent/templates/controller-configmap.yaml +++ b/helm/kagent/templates/controller-configmap.yaml @@ -51,6 +51,30 @@ data: OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: {{ .Values.otel.logging.exporter.otlp.timeout | quote }} {{- end }} {{- end }} + {{- if .Values.otel.tracing.exporter.otlp.headers }} + {{- $traceHeaders := list }} + {{- range $k := keys .Values.otel.tracing.exporter.otlp.headers | sortAlpha }} + {{- $traceHeaders = append $traceHeaders (printf "%s=%s" $k (index $.Values.otel.tracing.exporter.otlp.headers $k | toString)) }} + {{- end }} + OTEL_EXPORTER_OTLP_TRACES_HEADERS: {{ join "," $traceHeaders | quote }} + {{- end }} + {{- if .Values.otel.logging.exporter.otlp.headers }} + {{- $logHeaders := list }} + {{- range $k := keys .Values.otel.logging.exporter.otlp.headers | sortAlpha }} + {{- $logHeaders = append $logHeaders (printf "%s=%s" $k (index $.Values.otel.logging.exporter.otlp.headers $k | toString)) }} + {{- end }} + OTEL_EXPORTER_OTLP_LOGS_HEADERS: {{ join "," $logHeaders | quote }} + {{- end }} + {{- if .Values.otel.serviceName }} + OTEL_SERVICE_NAME: {{ .Values.otel.serviceName | quote }} + {{- end }} + {{- if .Values.otel.resourceAttributes }} + {{- $resourceAttrs := list }} + {{- range $k := keys .Values.otel.resourceAttributes | sortAlpha }} + {{- $resourceAttrs = append $resourceAttrs (printf "%s=%s" $k (index $.Values.otel.resourceAttributes $k | toString)) }} + {{- end }} + OTEL_RESOURCE_ATTRIBUTES: {{ join "," $resourceAttrs | quote }} + {{- end }} {{- if .Values.proxy.url }} PROXY_URL: {{ .Values.proxy.url | quote }} {{- end }} diff --git a/helm/kagent/tests/controller-deployment_test.yaml b/helm/kagent/tests/controller-deployment_test.yaml index 81832fd68..b4a08a1ae 100644 --- a/helm/kagent/tests/controller-deployment_test.yaml +++ b/helm/kagent/tests/controller-deployment_test.yaml @@ -166,6 +166,63 @@ tests: path: data.A2A_BASE_URL value: "https://kagent.example.com" + - it: should not set optional OTEL env vars by default + template: controller-configmap.yaml + asserts: + - notExists: + path: data.OTEL_SERVICE_NAME + - notExists: + path: data.OTEL_RESOURCE_ATTRIBUTES + - notExists: + path: data.OTEL_EXPORTER_OTLP_TRACES_HEADERS + - notExists: + path: data.OTEL_EXPORTER_OTLP_LOGS_HEADERS + + - it: should set OTEL_SERVICE_NAME when configured + template: controller-configmap.yaml + set: + otel: + serviceName: "kagent-controller-prod" + asserts: + - equal: + path: data.OTEL_SERVICE_NAME + value: "kagent-controller-prod" + + - it: should render OTEL_RESOURCE_ATTRIBUTES as sorted key=value pairs + template: controller-configmap.yaml + set: + otel: + resourceAttributes: + service.namespace: kagent + deployment.environment: production + asserts: + - equal: + path: data.OTEL_RESOURCE_ATTRIBUTES + value: "deployment.environment=production,service.namespace=kagent" + + - it: should render OTLP traces and logs headers as sorted key=value pairs + template: controller-configmap.yaml + set: + otel: + tracing: + exporter: + otlp: + headers: + x-honeycomb-team: trace-key + x-honeycomb-dataset: kagent + logging: + exporter: + otlp: + headers: + authorization: "Basic bG9ncw==" + asserts: + - equal: + path: data.OTEL_EXPORTER_OTLP_TRACES_HEADERS + value: "x-honeycomb-dataset=kagent,x-honeycomb-team=trace-key" + - equal: + path: data.OTEL_EXPORTER_OTLP_LOGS_HEADERS + value: "authorization=Basic bG9ncw==" + - it: should set PROXY_URL when set template: controller-configmap.yaml set: diff --git a/helm/kagent/values.yaml b/helm/kagent/values.yaml index e8a29fc60..447dd761c 100644 --- a/helm/kagent/values.yaml +++ b/helm/kagent/values.yaml @@ -917,6 +917,13 @@ oauth2-proxy: # ============================================================================== otel: + # -- Logical service name reported to the OTLP backend, rendered as OTEL_SERVICE_NAME. + # @default -- "" (SDK default) + serviceName: "" + # -- Additional OpenTelemetry resource attributes, rendered as a comma-joined + # key=value list in OTEL_RESOURCE_ATTRIBUTES. + # Example: { deployment.environment: production, service.namespace: kagent } + resourceAttributes: {} tracing: enabled: false exporter: @@ -925,6 +932,11 @@ otel: protocol: "grpc" timeout: 15000 # milliseconds insecure: true + # -- Headers sent with OTLP trace export requests, rendered as a comma-joined + # key=value list in OTEL_EXPORTER_OTLP_TRACES_HEADERS. + # Values land in a ConfigMap in plain text: for secrets (API keys, tokens), + # inject the env var from a Secret via controller.env / envFrom instead. + headers: {} logging: enabled: false exporter: @@ -932,3 +944,8 @@ otel: endpoint: "" timeout: 15000 # milliseconds insecure: true + # -- Headers sent with OTLP log export requests, rendered as a comma-joined + # key=value list in OTEL_EXPORTER_OTLP_LOGS_HEADERS. + # Values land in a ConfigMap in plain text: for secrets (API keys, tokens), + # inject the env var from a Secret via controller.env / envFrom instead. + headers: {} From 99ceaec24dd8a8e1e25a40f5d125b0075c26916f Mon Sep 17 00:00:00 2001 From: shekhargit1912 Date: Tue, 21 Jul 2026 23:29:54 +0530 Subject: [PATCH 2/2] docs(helm): warn that otel values propagate into every agent pod collectOtelEnvFromProcess() copies every OTEL_ env var from the controller's own process into every agent pod it creates, so otel.serviceName / otel.resourceAttributes relabel agent telemetry too, and OTLP header secrets set here land in plain text on every agent pod, not just the controller. Addresses review feedback from @mesutoezdil on #2272. Signed-off-by: shekhargit1912 --- helm/kagent/values.yaml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/helm/kagent/values.yaml b/helm/kagent/values.yaml index 447dd761c..36a1701ed 100644 --- a/helm/kagent/values.yaml +++ b/helm/kagent/values.yaml @@ -918,10 +918,16 @@ oauth2-proxy: otel: # -- Logical service name reported to the OTLP backend, rendered as OTEL_SERVICE_NAME. + # WARNING: the controller copies every OTEL_ env var from its own process into + # every agent pod it creates (see collectOtelEnvFromProcess), so setting this + # also relabels the service name reported by every agent's telemetry, not just + # the controller's. # @default -- "" (SDK default) serviceName: "" # -- Additional OpenTelemetry resource attributes, rendered as a comma-joined # key=value list in OTEL_RESOURCE_ATTRIBUTES. + # WARNING: like serviceName, this is copied into every agent pod's environment + # as well as the controller's. # Example: { deployment.environment: production, service.namespace: kagent } resourceAttributes: {} tracing: @@ -934,8 +940,11 @@ otel: insecure: true # -- Headers sent with OTLP trace export requests, rendered as a comma-joined # key=value list in OTEL_EXPORTER_OTLP_TRACES_HEADERS. - # Values land in a ConfigMap in plain text: for secrets (API keys, tokens), - # inject the env var from a Secret via controller.env / envFrom instead. + # WARNING: values land in the controller ConfigMap in plain text, AND the + # controller copies every OTEL_ env var into every agent pod it creates, so + # any secret set here (API keys, tokens) is exposed in plain text on every + # agent pod too, not just the controller. Prefer injecting such headers via + # controller.env / envFrom from a Secret instead. headers: {} logging: enabled: false @@ -946,6 +955,9 @@ otel: insecure: true # -- Headers sent with OTLP log export requests, rendered as a comma-joined # key=value list in OTEL_EXPORTER_OTLP_LOGS_HEADERS. - # Values land in a ConfigMap in plain text: for secrets (API keys, tokens), - # inject the env var from a Secret via controller.env / envFrom instead. + # WARNING: values land in the controller ConfigMap in plain text, AND the + # controller copies every OTEL_ env var into every agent pod it creates, so + # any secret set here (API keys, tokens) is exposed in plain text on every + # agent pod too, not just the controller. Prefer injecting such headers via + # controller.env / envFrom from a Secret instead. headers: {}