[HDX-4879] Restore inline custom OTEL collector config support

.changeset/otel-custom-config-support.md

@@ -0,0 +1,15 @@
+---
+"helm-charts": minor
+---
+
+feat: restore inline custom OTEL collector config support (HDX-4879)
+
+Adds `global.otelCollector.customConfig` to the clickstack chart, restoring
+the `otel.customConfig` capability that was lost in the v3 migration to the
+official OpenTelemetry Collector subchart. When set, the config is rendered
+into the `clickstack-otel-custom-config` ConfigMap, mounted at
+`/etc/otelcol-contrib/custom/custom.config.yaml`, and exposed to the
+collector via the `CUSTOM_OTELCOL_CONFIG_FILE` environment variable so it is
+merged on top of the built-in configuration in both OpAMP supervisor and
+standalone mode. Collector pods restart automatically when the config
+changes via a `checksum/custom-config` pod annotation.

.github/workflows/helm-test.yaml

@@ -43,6 +43,8 @@ jobs:
             -f examples/alb-ingress/values.yaml >/dev/null
           helm template clickstack-example charts/clickstack \
             -f examples/api-only/values.yaml >/dev/null
+          helm template clickstack-example charts/clickstack \
+            -f examples/otel-custom-config/values.yaml >/dev/null
 
       - name: Run helm-unittest
         run: |

charts/clickstack/templates/hyperdx/configmap.yaml

@@ -8,3 +8,6 @@ data:
   {{- range $k, $v := .Values.hyperdx.config }}
   {{ $k }}: {{ tpl (toString $v) $ | quote }}
   {{- end }}
+  {{- if and .Values.global.otelCollector.customConfig (not (hasKey .Values.hyperdx.config "CUSTOM_OTELCOL_CONFIG_FILE")) }}
+  CUSTOM_OTELCOL_CONFIG_FILE: "/etc/otelcol-contrib/custom/custom.config.yaml"
+  {{- end }}

charts/clickstack/templates/otel-collector/custom-config.yaml

@@ -0,0 +1,12 @@
+{{- if and (index .Values "otel-collector" "enabled") .Values.global.otelCollector.customConfig }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: clickstack-otel-custom-config
+  labels:
+    {{- include "clickstack.labels" . | nindent 4 }}
+    app.kubernetes.io/component: otel-collector
+data:
+  custom.config.yaml: |
+    {{- .Values.global.otelCollector.customConfig | nindent 4 }}
+{{- end }}

charts/clickstack/tests/otel-collector-custom-config_test.yaml

@@ -1,19 +1,136 @@
-suite: Test ClickStack ConfigMap Rendering
+suite: Test OTEL Collector Custom Config
 templates:
+  - otel-collector/custom-config.yaml
   - hyperdx/configmap.yaml
-
+  # configmap.yaml must be listed alongside deployment.yaml because the
+  # subchart deployment includes it for its own checksum annotation
+  - charts/otel-collector/templates/deployment.yaml
+  - charts/otel-collector/templates/configmap.yaml
 tests:
-  - it: should render ConfigMap with all expected keys
+  - it: should not render the custom config ConfigMap by default
+    template: otel-collector/custom-config.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should render the custom config ConfigMap when customConfig is set
+    template: otel-collector/custom-config.yaml
+    set:
+      global:
+        otelCollector:
+          customConfig: |
+            receivers:
+              hostmetrics:
+                collection_interval: 5s
+                scrapers:
+                  cpu:
+            service:
+              pipelines:
+                metrics/hostmetrics:
+                  receivers: [hostmetrics]
+                  processors: [memory_limiter, batch]
+                  exporters: [clickhouse]
     asserts:
       - hasDocuments:
           count: 1
       - isKind:
           of: ConfigMap
-      - isNotNull:
-          path: data.CLICKHOUSE_ENDPOINT
-      - isNotNull:
-          path: data.OPAMP_SERVER_URL
-      - isNotNull:
-          path: data.MONGO_URI
-      - isNotNull:
-          path: data.FRONTEND_URL
+      - equal:
+          path: metadata.name
+          value: clickstack-otel-custom-config
+      - equal:
+          path: metadata.labels["app.kubernetes.io/component"]
+          value: otel-collector
+      - matchRegex:
+          path: data["custom.config.yaml"]
+          pattern: "receivers:\n  hostmetrics:\n    collection_interval: 5s"
+      - matchRegex:
+          path: data["custom.config.yaml"]
+          pattern: "metrics/hostmetrics:"
+
+  - it: should not render the custom config ConfigMap when otel-collector is disabled
+    template: otel-collector/custom-config.yaml
+    set:
+      otel-collector:
+        enabled: false
+      global:
+        otelCollector:
+          customConfig: |
+            receivers:
+              hostmetrics:
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should not set CUSTOM_OTELCOL_CONFIG_FILE in clickstack-config by default
+    template: hyperdx/configmap.yaml
+    asserts:
+      - notExists:
+          path: data.CUSTOM_OTELCOL_CONFIG_FILE
+
+  - it: should set CUSTOM_OTELCOL_CONFIG_FILE in clickstack-config when customConfig is set
+    template: hyperdx/configmap.yaml
+    set:
+      global:
+        otelCollector:
+          customConfig: |
+            receivers:
+              hostmetrics:
+    asserts:
+      - equal:
+          path: data.CUSTOM_OTELCOL_CONFIG_FILE
+          value: "/etc/otelcol-contrib/custom/custom.config.yaml"
+
+  - it: should let hyperdx.config override CUSTOM_OTELCOL_CONFIG_FILE
+    template: hyperdx/configmap.yaml
+    set:
+      global:
+        otelCollector:
+          customConfig: |
+            receivers:
+              hostmetrics:
+      hyperdx:
+        config:
+          CUSTOM_OTELCOL_CONFIG_FILE: "/custom/path/config.yaml"
+    asserts:
+      - equal:
+          path: data.CUSTOM_OTELCOL_CONFIG_FILE
+          value: "/custom/path/config.yaml"
+
+  - it: should mount the optional custom config volume on the collector deployment
+    template: charts/otel-collector/templates/deployment.yaml
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: custom-config
+            configMap:
+              name: clickstack-otel-custom-config
+              optional: true
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: custom-config
+            mountPath: /etc/otelcol-contrib/custom
+            readOnly: true
+
+  - it: should carry a stable custom-config checksum annotation by default
+    template: charts/otel-collector/templates/deployment.yaml
+    asserts:
+      # sha256 of the default empty customConfig
+      - equal:
+          path: spec.template.metadata.annotations["checksum/custom-config"]
+          value: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+
+  - it: should change the custom-config checksum annotation when customConfig is set
+    template: charts/otel-collector/templates/deployment.yaml
+    set:
+      global:
+        otelCollector:
+          customConfig: |
+            receivers:
+              hostmetrics:
+    asserts:
+      - notEqual:
+          path: spec.template.metadata.annotations["checksum/custom-config"]
+          value: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

charts/clickstack/values.yaml

@@ -8,6 +8,36 @@ global:
   #   - name: docker-hub-secret
   imagePullSecrets: []
 
+  otelCollector:
+    # Custom OTEL Collector configuration, merged on top of the collector's
+    # built-in config (works in both OpAMP supervisor and standalone mode).
+    # Provide the configuration as a multi-line YAML string. When set, it is
+    # rendered into the `clickstack-otel-custom-config` ConfigMap, mounted at
+    # /etc/otelcol-contrib/custom/custom.config.yaml, and the
+    # CUSTOM_OTELCOL_CONFIG_FILE environment variable points the collector
+    # to it. Changing the value automatically restarts the collector pods.
+    #
+    # NOTE: the OTel confmap merges maps leaf-by-leaf. To override a built-in
+    # processor (e.g. memory_limiter), define a new one under a different name
+    # and re-declare the pipeline `processors:` lists to reference it.
+    #
+    # Example:
+    # customConfig: |
+    #   receivers:
+    #     hostmetrics:
+    #       collection_interval: 5s
+    #       scrapers:
+    #         cpu:
+    #         load:
+    #         memory:
+    #   service:
+    #     pipelines:
+    #       metrics/hostmetrics:
+    #         receivers: [hostmetrics]
+    #         processors: [memory_limiter, batch]
+    #         exporters: [clickhouse]
+    customConfig: ""
+
 hyperdx:
   # Ports shared across Deployment, Service, ConfigMap, and Ingress
   ports:
@@ -383,6 +413,25 @@ otel-collector:
         name: clickstack-config
     - secretRef:
         name: clickstack-secret
+  # Roll collector pods when global.otelCollector.customConfig changes.
+  # Merges with any user-provided podAnnotations.
+  podAnnotations:
+    checksum/custom-config: '{{ .Values.global.otelCollector.customConfig | default "" | sha256sum }}'
+  # Mount for the optional custom config ConfigMap rendered from
+  # global.otelCollector.customConfig. The volume is marked optional so it is
+  # a no-op when no custom config is set.
+  # NOTE: if you override extraVolumes/extraVolumeMounts yourself, Helm
+  # replaces these lists entirely -- re-include the entries below to keep
+  # global.otelCollector.customConfig working.
+  extraVolumes:
+    - name: custom-config
+      configMap:
+        name: clickstack-otel-custom-config
+        optional: true
+  extraVolumeMounts:
+    - name: custom-config
+      mountPath: /etc/otelcol-contrib/custom
+      readOnly: true
   ports:
     otlp:
       enabled: true

docs/UPGRADE.md

@@ -362,6 +362,29 @@ otel-collector:
 
 See the [OpenTelemetry Collector Helm chart](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-collector) for all available subchart values.
 
+To provide a custom collector configuration (previously `otel.customConfig`), use `global.otelCollector.customConfig`. The chart renders it into a ConfigMap, mounts it at `/etc/otelcol-contrib/custom/custom.config.yaml`, and sets `CUSTOM_OTELCOL_CONFIG_FILE` so the collector merges it on top of its built-in config:
+
+```yaml
+global:
+  otelCollector:
+    customConfig: |
+      receivers:
+        hostmetrics:
+          collection_interval: 5s
+          scrapers:
+            cpu:
+            load:
+            memory:
+      service:
+        pipelines:
+          metrics/hostmetrics:
+            receivers: [hostmetrics]
+            processors: [memory_limiter, batch]
+            exporters: [clickhouse]
+```
+
+Note: the subchart's native `config:`/`alternateConfig:` values are **not** honored by the ClickStack collector image -- its entrypoint runs the collector (or OpAMP supervisor) with its own baked-in config files and ignores the chart-generated `relay.yaml`. Use `global.otelCollector.customConfig` instead.
+
 ## Unchanged Values
 
 The following sections are **not affected** by this migration:

examples/otel-custom-config/README.md

@@ -0,0 +1,44 @@
+# ClickStack with a custom OTEL Collector configuration
+
+This example shows how to layer a custom OpenTelemetry Collector configuration on top of the built-in ClickStack collector config using `global.otelCollector.customConfig`.
+
+## How it works
+
+When `global.otelCollector.customConfig` is set, the chart:
+
+1. Renders the value into the `clickstack-otel-custom-config` ConfigMap (key `custom.config.yaml`)
+2. Mounts it into the collector pod at `/etc/otelcol-contrib/custom/custom.config.yaml` (the mount is pre-wired via `otel-collector.extraVolumes`/`extraVolumeMounts` and marked `optional`, so it is a no-op when unset)
+3. Adds `CUSTOM_OTELCOL_CONFIG_FILE=/etc/otelcol-contrib/custom/custom.config.yaml` to the shared `clickstack-config` ConfigMap, which the collector image's entrypoint picks up in both OpAMP supervisor mode (the default) and standalone mode
+4. Stamps a `checksum/custom-config` pod annotation so collector pods restart automatically when the config changes
+
+## What's included
+
+The `values.yaml` in this directory:
+
+- Adds a `hostmetrics` receiver and a new `metrics/hostmetrics` pipeline exporting to ClickHouse
+- Swaps the built-in `memory_limiter` processor for a percentage-based `memory_limiter/custom` that scales with the pod's memory allocation
+
+## Merge semantics
+
+The custom config is merged **leaf-by-leaf** on top of the collector's built-in configuration by the OTel confmap resolver:
+
+- New receivers/processors/exporters/pipelines are added
+- To change a built-in component's settings, prefer defining a **new component under a different name** (e.g. `memory_limiter/custom`) and re-declaring the pipeline `processors:` lists that should use it. Leaf-merging into some built-in components (notably `memory_limiter`) does not behave as expected
+- Pipelines you don't re-declare keep their built-in configuration
+
+## Caveats
+
+- The subchart's native `config:`/`alternateConfig:` values are **not** honored by the ClickStack collector image — its entrypoint ignores the chart-generated `relay.yaml`. Always use `global.otelCollector.customConfig`.
+- If you override `otel-collector.extraVolumes` or `otel-collector.extraVolumeMounts` yourself, Helm replaces the pre-wired lists entirely — re-include the `custom-config` entries from the chart's default `values.yaml` to keep this feature working.
+
+## Usage
+
+```bash
+helm install my-clickstack clickstack/clickstack -f values.yaml
+```
+
+Verify the merged config took effect (supervisor mode) by checking the effective config inside the collector pod:
+
+```bash
+kubectl exec deploy/my-clickstack-otel-collector -- cat /etc/otel/supervisor-data/effective.yaml
+```

examples/otel-custom-config/values.yaml

@@ -0,0 +1,55 @@
+# ClickStack with a custom OTEL Collector configuration
+#
+# This example extends the built-in collector configuration with a custom
+# config layer via global.otelCollector.customConfig. The chart renders the
+# value into the `clickstack-otel-custom-config` ConfigMap, mounts it at
+# /etc/otelcol-contrib/custom/custom.config.yaml, and sets the
+# CUSTOM_OTELCOL_CONFIG_FILE environment variable so the collector merges it
+# on top of its built-in config. This works in both OpAMP supervisor mode
+# (the default) and standalone mode. Collector pods restart automatically
+# when the config changes.
+#
+# The OTel confmap merges maps leaf-by-leaf. To override a built-in
+# processor (e.g. memory_limiter), define a new one under a different name
+# and re-declare the pipeline `processors:` lists to reference it.
+#
+# Usage:
+#   helm install my-clickstack clickstack/clickstack -f values.yaml
+
+global:
+  otelCollector:
+    customConfig: |
+      receivers:
+        # Collect host metrics from the collector's node
+        hostmetrics:
+          collection_interval: 5s
+          scrapers:
+            cpu:
+            load:
+            memory:
+
+      processors:
+        # Example: swap the built-in memory_limiter (sized for a ~2 GiB
+        # container) for a percentage-based one that scales with the pod's
+        # memory allocation.
+        memory_limiter/custom:
+          check_interval: 5s
+          limit_percentage: 75
+          spike_limit_percentage: 25
+
+      service:
+        pipelines:
+          # New pipeline feeding host metrics into ClickHouse
+          metrics/hostmetrics:
+            receivers: [hostmetrics]
+            processors: [memory_limiter, batch]
+            exporters: [clickhouse]
+          # Re-declare built-in pipelines to use the custom memory limiter
+          traces:
+            processors: [memory_limiter/custom, batch]
+          metrics:
+            processors: [memory_limiter/custom, batch]
+          logs/out-default:
+            processors: [memory_limiter/custom, transform, batch]
+          logs/out-rrweb:
+            processors: [memory_limiter/custom, batch]