From b6854ee7ffa2a6bcea64bb4d572bb61bb4de1ea8 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:35:32 -0700 Subject: [PATCH] feat: restore inline custom OTEL collector config support (HDX-4879) Adds global.otelCollector.customConfig, restoring the otel.customConfig capability lost in the v3 migration to the official OpenTelemetry Collector subchart. When set, the config is rendered into the clickstack-otel-custom-config ConfigMap, mounted at /etc/otelcol-contrib/custom/custom.config.yaml, and exposed via the CUSTOM_OTELCOL_CONFIG_FILE env var (through the shared clickstack-config ConfigMap) so the collector image merges it on top of its built-in config in both OpAMP supervisor and standalone mode. A checksum/custom-config pod annotation rolls collector pods on change. --- .changeset/otel-custom-config-support.md | 15 ++ .github/workflows/helm-test.yaml | 2 + .../templates/hyperdx/configmap.yaml | 3 + .../otel-collector/custom-config.yaml | 12 ++ .../otel-collector-custom-config_test.yaml | 139 ++++++++++++++++-- charts/clickstack/values.yaml | 49 ++++++ docs/UPGRADE.md | 23 +++ examples/otel-custom-config/README.md | 44 ++++++ examples/otel-custom-config/values.yaml | 55 +++++++ 9 files changed, 331 insertions(+), 11 deletions(-) create mode 100644 .changeset/otel-custom-config-support.md create mode 100644 charts/clickstack/templates/otel-collector/custom-config.yaml create mode 100644 examples/otel-custom-config/README.md create mode 100644 examples/otel-custom-config/values.yaml diff --git a/.changeset/otel-custom-config-support.md b/.changeset/otel-custom-config-support.md new file mode 100644 index 00000000..72ea06d0 --- /dev/null +++ b/.changeset/otel-custom-config-support.md @@ -0,0 +1,15 @@ +--- +"helm-charts": minor +--- + +feat: restore inline custom OTEL collector config support (HDX-4879) + +Adds `global.otelCollector.customConfig` to the clickstack chart, restoring +the `otel.customConfig` capability that was lost in the v3 migration to the +official OpenTelemetry Collector subchart. When set, the config is rendered +into the `clickstack-otel-custom-config` ConfigMap, mounted at +`/etc/otelcol-contrib/custom/custom.config.yaml`, and exposed to the +collector via the `CUSTOM_OTELCOL_CONFIG_FILE` environment variable so it is +merged on top of the built-in configuration in both OpAMP supervisor and +standalone mode. Collector pods restart automatically when the config +changes via a `checksum/custom-config` pod annotation. diff --git a/.github/workflows/helm-test.yaml b/.github/workflows/helm-test.yaml index c8488014..48bdd804 100644 --- a/.github/workflows/helm-test.yaml +++ b/.github/workflows/helm-test.yaml @@ -43,6 +43,8 @@ jobs: -f examples/alb-ingress/values.yaml >/dev/null helm template clickstack-example charts/clickstack \ -f examples/api-only/values.yaml >/dev/null + helm template clickstack-example charts/clickstack \ + -f examples/otel-custom-config/values.yaml >/dev/null - name: Run helm-unittest run: | diff --git a/charts/clickstack/templates/hyperdx/configmap.yaml b/charts/clickstack/templates/hyperdx/configmap.yaml index 05813910..4b37b3aa 100644 --- a/charts/clickstack/templates/hyperdx/configmap.yaml +++ b/charts/clickstack/templates/hyperdx/configmap.yaml @@ -8,3 +8,6 @@ data: {{- range $k, $v := .Values.hyperdx.config }} {{ $k }}: {{ tpl (toString $v) $ | quote }} {{- end }} + {{- if and .Values.global.otelCollector.customConfig (not (hasKey .Values.hyperdx.config "CUSTOM_OTELCOL_CONFIG_FILE")) }} + CUSTOM_OTELCOL_CONFIG_FILE: "/etc/otelcol-contrib/custom/custom.config.yaml" + {{- end }} diff --git a/charts/clickstack/templates/otel-collector/custom-config.yaml b/charts/clickstack/templates/otel-collector/custom-config.yaml new file mode 100644 index 00000000..39dd18f0 --- /dev/null +++ b/charts/clickstack/templates/otel-collector/custom-config.yaml @@ -0,0 +1,12 @@ +{{- if and (index .Values "otel-collector" "enabled") .Values.global.otelCollector.customConfig }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: clickstack-otel-custom-config + labels: + {{- include "clickstack.labels" . | nindent 4 }} + app.kubernetes.io/component: otel-collector +data: + custom.config.yaml: | + {{- .Values.global.otelCollector.customConfig | nindent 4 }} +{{- end }} diff --git a/charts/clickstack/tests/otel-collector-custom-config_test.yaml b/charts/clickstack/tests/otel-collector-custom-config_test.yaml index ac782741..8ffef998 100644 --- a/charts/clickstack/tests/otel-collector-custom-config_test.yaml +++ b/charts/clickstack/tests/otel-collector-custom-config_test.yaml @@ -1,19 +1,136 @@ -suite: Test ClickStack ConfigMap Rendering +suite: Test OTEL Collector Custom Config templates: + - otel-collector/custom-config.yaml - hyperdx/configmap.yaml - + # configmap.yaml must be listed alongside deployment.yaml because the + # subchart deployment includes it for its own checksum annotation + - charts/otel-collector/templates/deployment.yaml + - charts/otel-collector/templates/configmap.yaml tests: - - it: should render ConfigMap with all expected keys + - it: should not render the custom config ConfigMap by default + template: otel-collector/custom-config.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should render the custom config ConfigMap when customConfig is set + template: otel-collector/custom-config.yaml + set: + global: + otelCollector: + customConfig: | + receivers: + hostmetrics: + collection_interval: 5s + scrapers: + cpu: + service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics] + processors: [memory_limiter, batch] + exporters: [clickhouse] asserts: - hasDocuments: count: 1 - isKind: of: ConfigMap - - isNotNull: - path: data.CLICKHOUSE_ENDPOINT - - isNotNull: - path: data.OPAMP_SERVER_URL - - isNotNull: - path: data.MONGO_URI - - isNotNull: - path: data.FRONTEND_URL + - equal: + path: metadata.name + value: clickstack-otel-custom-config + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: otel-collector + - matchRegex: + path: data["custom.config.yaml"] + pattern: "receivers:\n hostmetrics:\n collection_interval: 5s" + - matchRegex: + path: data["custom.config.yaml"] + pattern: "metrics/hostmetrics:" + + - it: should not render the custom config ConfigMap when otel-collector is disabled + template: otel-collector/custom-config.yaml + set: + otel-collector: + enabled: false + global: + otelCollector: + customConfig: | + receivers: + hostmetrics: + asserts: + - hasDocuments: + count: 0 + + - it: should not set CUSTOM_OTELCOL_CONFIG_FILE in clickstack-config by default + template: hyperdx/configmap.yaml + asserts: + - notExists: + path: data.CUSTOM_OTELCOL_CONFIG_FILE + + - it: should set CUSTOM_OTELCOL_CONFIG_FILE in clickstack-config when customConfig is set + template: hyperdx/configmap.yaml + set: + global: + otelCollector: + customConfig: | + receivers: + hostmetrics: + asserts: + - equal: + path: data.CUSTOM_OTELCOL_CONFIG_FILE + value: "/etc/otelcol-contrib/custom/custom.config.yaml" + + - it: should let hyperdx.config override CUSTOM_OTELCOL_CONFIG_FILE + template: hyperdx/configmap.yaml + set: + global: + otelCollector: + customConfig: | + receivers: + hostmetrics: + hyperdx: + config: + CUSTOM_OTELCOL_CONFIG_FILE: "/custom/path/config.yaml" + asserts: + - equal: + path: data.CUSTOM_OTELCOL_CONFIG_FILE + value: "/custom/path/config.yaml" + + - it: should mount the optional custom config volume on the collector deployment + template: charts/otel-collector/templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: custom-config + configMap: + name: clickstack-otel-custom-config + optional: true + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: custom-config + mountPath: /etc/otelcol-contrib/custom + readOnly: true + + - it: should carry a stable custom-config checksum annotation by default + template: charts/otel-collector/templates/deployment.yaml + asserts: + # sha256 of the default empty customConfig + - equal: + path: spec.template.metadata.annotations["checksum/custom-config"] + value: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + + - it: should change the custom-config checksum annotation when customConfig is set + template: charts/otel-collector/templates/deployment.yaml + set: + global: + otelCollector: + customConfig: | + receivers: + hostmetrics: + asserts: + - notEqual: + path: spec.template.metadata.annotations["checksum/custom-config"] + value: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 diff --git a/charts/clickstack/values.yaml b/charts/clickstack/values.yaml index 2dfcb2fe..46716360 100644 --- a/charts/clickstack/values.yaml +++ b/charts/clickstack/values.yaml @@ -8,6 +8,36 @@ global: # - name: docker-hub-secret imagePullSecrets: [] + otelCollector: + # Custom OTEL Collector configuration, merged on top of the collector's + # built-in config (works in both OpAMP supervisor and standalone mode). + # Provide the configuration as a multi-line YAML string. When set, it is + # rendered into the `clickstack-otel-custom-config` ConfigMap, mounted at + # /etc/otelcol-contrib/custom/custom.config.yaml, and the + # CUSTOM_OTELCOL_CONFIG_FILE environment variable points the collector + # to it. Changing the value automatically restarts the collector pods. + # + # NOTE: the OTel confmap merges maps leaf-by-leaf. To override a built-in + # processor (e.g. memory_limiter), define a new one under a different name + # and re-declare the pipeline `processors:` lists to reference it. + # + # Example: + # customConfig: | + # receivers: + # hostmetrics: + # collection_interval: 5s + # scrapers: + # cpu: + # load: + # memory: + # service: + # pipelines: + # metrics/hostmetrics: + # receivers: [hostmetrics] + # processors: [memory_limiter, batch] + # exporters: [clickhouse] + customConfig: "" + hyperdx: # Ports shared across Deployment, Service, ConfigMap, and Ingress ports: @@ -383,6 +413,25 @@ otel-collector: name: clickstack-config - secretRef: name: clickstack-secret + # Roll collector pods when global.otelCollector.customConfig changes. + # Merges with any user-provided podAnnotations. + podAnnotations: + checksum/custom-config: '{{ .Values.global.otelCollector.customConfig | default "" | sha256sum }}' + # Mount for the optional custom config ConfigMap rendered from + # global.otelCollector.customConfig. The volume is marked optional so it is + # a no-op when no custom config is set. + # NOTE: if you override extraVolumes/extraVolumeMounts yourself, Helm + # replaces these lists entirely -- re-include the entries below to keep + # global.otelCollector.customConfig working. + extraVolumes: + - name: custom-config + configMap: + name: clickstack-otel-custom-config + optional: true + extraVolumeMounts: + - name: custom-config + mountPath: /etc/otelcol-contrib/custom + readOnly: true ports: otlp: enabled: true diff --git a/docs/UPGRADE.md b/docs/UPGRADE.md index 1e17c2ff..86668477 100644 --- a/docs/UPGRADE.md +++ b/docs/UPGRADE.md @@ -362,6 +362,29 @@ otel-collector: See the [OpenTelemetry Collector Helm chart](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector) for all available subchart values. +To provide a custom collector configuration (previously `otel.customConfig`), use `global.otelCollector.customConfig`. The chart renders it into a ConfigMap, mounts it at `/etc/otelcol-contrib/custom/custom.config.yaml`, and sets `CUSTOM_OTELCOL_CONFIG_FILE` so the collector merges it on top of its built-in config: + +```yaml +global: + otelCollector: + customConfig: | + receivers: + hostmetrics: + collection_interval: 5s + scrapers: + cpu: + load: + memory: + service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics] + processors: [memory_limiter, batch] + exporters: [clickhouse] +``` + +Note: the subchart's native `config:`/`alternateConfig:` values are **not** honored by the ClickStack collector image -- its entrypoint runs the collector (or OpAMP supervisor) with its own baked-in config files and ignores the chart-generated `relay.yaml`. Use `global.otelCollector.customConfig` instead. + ## Unchanged Values The following sections are **not affected** by this migration: diff --git a/examples/otel-custom-config/README.md b/examples/otel-custom-config/README.md new file mode 100644 index 00000000..f91b200a --- /dev/null +++ b/examples/otel-custom-config/README.md @@ -0,0 +1,44 @@ +# ClickStack with a custom OTEL Collector configuration + +This example shows how to layer a custom OpenTelemetry Collector configuration on top of the built-in ClickStack collector config using `global.otelCollector.customConfig`. + +## How it works + +When `global.otelCollector.customConfig` is set, the chart: + +1. Renders the value into the `clickstack-otel-custom-config` ConfigMap (key `custom.config.yaml`) +2. Mounts it into the collector pod at `/etc/otelcol-contrib/custom/custom.config.yaml` (the mount is pre-wired via `otel-collector.extraVolumes`/`extraVolumeMounts` and marked `optional`, so it is a no-op when unset) +3. Adds `CUSTOM_OTELCOL_CONFIG_FILE=/etc/otelcol-contrib/custom/custom.config.yaml` to the shared `clickstack-config` ConfigMap, which the collector image's entrypoint picks up in both OpAMP supervisor mode (the default) and standalone mode +4. Stamps a `checksum/custom-config` pod annotation so collector pods restart automatically when the config changes + +## What's included + +The `values.yaml` in this directory: + +- Adds a `hostmetrics` receiver and a new `metrics/hostmetrics` pipeline exporting to ClickHouse +- Swaps the built-in `memory_limiter` processor for a percentage-based `memory_limiter/custom` that scales with the pod's memory allocation + +## Merge semantics + +The custom config is merged **leaf-by-leaf** on top of the collector's built-in configuration by the OTel confmap resolver: + +- New receivers/processors/exporters/pipelines are added +- To change a built-in component's settings, prefer defining a **new component under a different name** (e.g. `memory_limiter/custom`) and re-declaring the pipeline `processors:` lists that should use it. Leaf-merging into some built-in components (notably `memory_limiter`) does not behave as expected +- Pipelines you don't re-declare keep their built-in configuration + +## Caveats + +- The subchart's native `config:`/`alternateConfig:` values are **not** honored by the ClickStack collector image — its entrypoint ignores the chart-generated `relay.yaml`. Always use `global.otelCollector.customConfig`. +- If you override `otel-collector.extraVolumes` or `otel-collector.extraVolumeMounts` yourself, Helm replaces the pre-wired lists entirely — re-include the `custom-config` entries from the chart's default `values.yaml` to keep this feature working. + +## Usage + +```bash +helm install my-clickstack clickstack/clickstack -f values.yaml +``` + +Verify the merged config took effect (supervisor mode) by checking the effective config inside the collector pod: + +```bash +kubectl exec deploy/my-clickstack-otel-collector -- cat /etc/otel/supervisor-data/effective.yaml +``` diff --git a/examples/otel-custom-config/values.yaml b/examples/otel-custom-config/values.yaml new file mode 100644 index 00000000..1a4220f6 --- /dev/null +++ b/examples/otel-custom-config/values.yaml @@ -0,0 +1,55 @@ +# ClickStack with a custom OTEL Collector configuration +# +# This example extends the built-in collector configuration with a custom +# config layer via global.otelCollector.customConfig. The chart renders the +# value into the `clickstack-otel-custom-config` ConfigMap, mounts it at +# /etc/otelcol-contrib/custom/custom.config.yaml, and sets the +# CUSTOM_OTELCOL_CONFIG_FILE environment variable so the collector merges it +# on top of its built-in config. This works in both OpAMP supervisor mode +# (the default) and standalone mode. Collector pods restart automatically +# when the config changes. +# +# The OTel confmap merges maps leaf-by-leaf. To override a built-in +# processor (e.g. memory_limiter), define a new one under a different name +# and re-declare the pipeline `processors:` lists to reference it. +# +# Usage: +# helm install my-clickstack clickstack/clickstack -f values.yaml + +global: + otelCollector: + customConfig: | + receivers: + # Collect host metrics from the collector's node + hostmetrics: + collection_interval: 5s + scrapers: + cpu: + load: + memory: + + processors: + # Example: swap the built-in memory_limiter (sized for a ~2 GiB + # container) for a percentage-based one that scales with the pod's + # memory allocation. + memory_limiter/custom: + check_interval: 5s + limit_percentage: 75 + spike_limit_percentage: 25 + + service: + pipelines: + # New pipeline feeding host metrics into ClickHouse + metrics/hostmetrics: + receivers: [hostmetrics] + processors: [memory_limiter, batch] + exporters: [clickhouse] + # Re-declare built-in pipelines to use the custom memory limiter + traces: + processors: [memory_limiter/custom, batch] + metrics: + processors: [memory_limiter/custom, batch] + logs/out-default: + processors: [memory_limiter/custom, transform, batch] + logs/out-rrweb: + processors: [memory_limiter/custom, batch]