From 3215f36b8f1f896420c7facda0dda31ec1056055 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Mon, 5 Oct 2026 17:01:42 +0000 Subject: [PATCH 1/6] feat(telemetry): support custom OTLP headers in telemetry configuration --- docs/cli/telemetry.md | 1 + docs/reference/configuration.md | 8 + packages/cli/src/config/settings.test.ts | 33 ++++ packages/cli/src/config/settingsSchema.ts | 5 + packages/core/src/config/config.test.ts | 26 ++++ packages/core/src/config/config.ts | 6 + packages/core/src/telemetry/config.test.ts | 171 +++++++++++++++++++++ packages/core/src/telemetry/config.ts | 168 ++++++++++++++++++++ packages/core/src/telemetry/index.ts | 1 + packages/core/src/telemetry/sdk.test.ts | 67 ++++++++ packages/core/src/telemetry/sdk.ts | 17 ++ schemas/settings.schema.json | 7 + 12 files changed, 510 insertions(+) diff --git a/docs/cli/telemetry.md b/docs/cli/telemetry.md index a1d44eeb235..49198b63167 100644 --- a/docs/cli/telemetry.md +++ b/docs/cli/telemetry.md @@ -42,6 +42,7 @@ Environment variables can override these settings. | `target` | `GEMINI_TELEMETRY_TARGET` | Where to send telemetry data | `"gcp"`/`"local"` | `"local"` | | `otlpEndpoint` | `GEMINI_TELEMETRY_OTLP_ENDPOINT` | OTLP collector endpoint | URL string | `http://localhost:4317` | | `otlpProtocol` | `GEMINI_TELEMETRY_OTLP_PROTOCOL` | OTLP transport protocol | `"grpc"`/`"http"` | `"grpc"` | +| `otlpHeaders` | `GEMINI_TELEMETRY_OTLP_HEADERS` | Custom headers for OTLP exporter requests | JSON/`key=value` | - | | `outfile` | `GEMINI_TELEMETRY_OUTFILE` | Save telemetry to file (overrides `otlpEndpoint`) | file path | - | | `logPrompts` | `GEMINI_TELEMETRY_LOG_PROMPTS` | Include prompts in telemetry logs | `true`/`false` | `true` | | `useCollector` | `GEMINI_TELEMETRY_USE_COLLECTOR` | Use external OTLP collector (advanced) | `true`/`false` | `false` | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 61d158ded90..0ccca52328e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -2550,6 +2550,9 @@ see [Telemetry](../cli/telemetry.md). - **`otlpEndpoint`** (string): The endpoint for the OTLP Exporter. - **`otlpProtocol`** (string): The protocol for the OTLP Exporter (`grpc` or `http`). + - **`otlpHeaders`** (object): Custom headers to send with OTLP Exporter + requests (supports environment variable references like `$VAR_NAME` or + `${VAR_NAME}`). - **`logPrompts`** (boolean): Whether or not to include the content of user prompts in the logs. - **`outfile`** (string): The file to write telemetry to when `target` is @@ -2766,6 +2769,11 @@ the `advanced.excludedEnvVars` setting in your `settings.json` file. - **`GEMINI_TELEMETRY_OTLP_PROTOCOL`**: - Sets the OTLP protocol (`grpc` or `http`). - Overrides the `telemetry.otlpProtocol` setting. +- **`GEMINI_TELEMETRY_OTLP_HEADERS`**: + - Sets custom headers for OTLP exporter requests (as a JSON object string or + comma/semicolon-separated `key=value` pairs; also falls back to + `OTEL_EXPORTER_OTLP_HEADERS`). + - Merges with and overrides headers in the `telemetry.otlpHeaders` setting. - **`GEMINI_TELEMETRY_LOG_PROMPTS`**: - Set to `true` or `1` to enable or disable logging of user prompts. Any other value is treated as disabling it. diff --git a/packages/cli/src/config/settings.test.ts b/packages/cli/src/config/settings.test.ts index b275de402ad..545a77363b0 100644 --- a/packages/cli/src/config/settings.test.ts +++ b/packages/cli/src/config/settings.test.ts @@ -903,6 +903,39 @@ describe('Settings Loading and Merging', () => { expect(settings.merged.telemetry?.enabled).toBe(false); }); + it('should load telemetry.otlpHeaders and resolve environment variables', () => { + process.env['TEST_OTLP_TOKEN'] = 'secret-token-value'; + try { + (mockFsExistsSync as Mock).mockImplementation( + (p: fs.PathLike) => + normalizePath(p) === normalizePath(USER_SETTINGS_PATH), + ); + const userSettingsContent = { + telemetry: { + enabled: true, + otlpHeaders: { + Authorization: 'Bearer ${TEST_OTLP_TOKEN}', + 'x-custom': 'static-val', + }, + }, + }; + (fs.readFileSync as Mock).mockImplementation( + (p: fs.PathOrFileDescriptor) => { + if (normalizePath(p) === normalizePath(USER_SETTINGS_PATH)) + return JSON.stringify(userSettingsContent); + return '{}'; + }, + ); + const settings = loadSettings(MOCK_WORKSPACE_DIR); + expect(settings.merged.telemetry?.otlpHeaders).toEqual({ + Authorization: 'Bearer secret-token-value', + 'x-custom': 'static-val', + }); + } finally { + delete process.env['TEST_OTLP_TOKEN']; + } + }); + it('should have telemetry as undefined if not in any settings file', () => { (mockFsExistsSync as Mock).mockReturnValue(false); // No settings files exist (fs.readFileSync as Mock).mockReturnValue('{}'); diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 0b480252240..7ce21d2a233 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3225,6 +3225,11 @@ export const SETTINGS_SCHEMA_DEFINITIONS: Record< description: 'Protocol for OTLP exporters.', enum: ['grpc', 'http'], }, + otlpHeaders: { + type: 'object', + description: 'Custom headers to send with OTLP exporter requests.', + additionalProperties: { type: 'string' }, + }, traces: { type: 'boolean', description: diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index e84c41f7d11..6775b67799f 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -1385,6 +1385,32 @@ describe('Server Config (config.ts)', () => { const config = new Config(paramsWithoutTelemetry); expect(config.getTelemetryOtlpProtocol()).toBe('grpc'); }); + + it('should return provided OTLP headers', () => { + const headers = { Authorization: 'Bearer token123' }; + const params: ConfigParameters = { + ...baseParams, + telemetry: { enabled: true, otlpHeaders: headers }, + }; + const config = new Config(params); + expect(config.getTelemetryOtlpHeaders()).toEqual(headers); + }); + + it('should return empty object for OTLP headers if not provided', () => { + const params: ConfigParameters = { + ...baseParams, + telemetry: { enabled: true }, + }; + const config = new Config(params); + expect(config.getTelemetryOtlpHeaders()).toEqual({}); + }); + + it('should return empty object for OTLP headers if telemetry object is not provided', () => { + const paramsWithoutTelemetry: ConfigParameters = { ...baseParams }; + delete paramsWithoutTelemetry.telemetry; + const config = new Config(paramsWithoutTelemetry); + expect(config.getTelemetryOtlpHeaders()).toEqual({}); + }); }); describe('UseRipgrep Configuration', () => { diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 6b8649e3500..5cb9d1c8220 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -222,6 +222,7 @@ export interface TelemetrySettings { target?: TelemetryTarget; otlpEndpoint?: string; otlpProtocol?: 'grpc' | 'http'; + otlpHeaders?: Record; logPrompts?: boolean; outfile?: string; useCollector?: boolean; @@ -1096,6 +1097,7 @@ export class Config implements McpContext, AgentLoopContext { target: params.telemetry?.target ?? DEFAULT_TELEMETRY_TARGET, otlpEndpoint: params.telemetry?.otlpEndpoint ?? DEFAULT_OTLP_ENDPOINT, otlpProtocol: params.telemetry?.otlpProtocol, + otlpHeaders: params.telemetry?.otlpHeaders, logPrompts: params.telemetry?.logPrompts ?? true, outfile: params.telemetry?.outfile, useCollector: params.telemetry?.useCollector, @@ -2922,6 +2924,10 @@ export class Config implements McpContext, AgentLoopContext { return this.telemetrySettings.otlpProtocol ?? 'grpc'; } + getTelemetryOtlpHeaders(): Record { + return this.telemetrySettings.otlpHeaders ?? {}; + } + getTelemetryTarget(): TelemetryTarget { return this.telemetrySettings.target ?? DEFAULT_TELEMETRY_TARGET; } diff --git a/packages/core/src/telemetry/config.test.ts b/packages/core/src/telemetry/config.test.ts index 232aa274667..f44933376af 100644 --- a/packages/core/src/telemetry/config.test.ts +++ b/packages/core/src/telemetry/config.test.ts @@ -7,6 +7,7 @@ import { describe, it, expect } from 'vitest'; import { parseBooleanEnvFlag, + parseOtlpHeaders, parseTelemetryTargetValue, resolveTelemetrySettings, } from './config.js'; @@ -53,6 +54,102 @@ describe('telemetry/config helpers', () => { }); }); + describe('parseOtlpHeaders', () => { + it('returns undefined for undefined or empty string', () => { + expect(parseOtlpHeaders(undefined)).toBeUndefined(); + expect(parseOtlpHeaders('')).toBeUndefined(); + expect(parseOtlpHeaders(' ')).toBeUndefined(); + }); + + it('parses JSON object format', () => { + const json = '{"Authorization":"Bearer token123","x-api-key":"abc"}'; + expect(parseOtlpHeaders(json)).toEqual({ + Authorization: 'Bearer token123', + 'x-api-key': 'abc', + }); + }); + + it('returns undefined for empty JSON object', () => { + expect(parseOtlpHeaders('{}')).toBeUndefined(); + }); + + it('returns undefined for malformed JSON or non-string values without falling back to key=value', () => { + expect(parseOtlpHeaders('{Authorization=Bearer token}')).toBeUndefined(); + expect(parseOtlpHeaders('{"key": 123}')).toBeUndefined(); + expect(parseOtlpHeaders('{"key": null}')).toBeUndefined(); + expect(parseOtlpHeaders('{"key": true}')).toBeUndefined(); + expect(parseOtlpHeaders('{"valid": "yes", "bad": 42}')).toBeUndefined(); + expect(parseOtlpHeaders('["a", "b"]')).toBeUndefined(); + }); + + it('parses comma-separated key=value pairs', () => { + expect( + parseOtlpHeaders('Authorization=Bearer token123,x-api-key=abc'), + ).toEqual({ + Authorization: 'Bearer token123', + 'x-api-key': 'abc', + }); + }); + + it('parses semicolon-separated key=value pairs', () => { + expect( + parseOtlpHeaders('Authorization=Bearer token123;x-api-key=abc'), + ).toEqual({ + Authorization: 'Bearer token123', + 'x-api-key': 'abc', + }); + }); + + it('handles values containing equals signs (e.g. base64)', () => { + expect(parseOtlpHeaders('Authorization=Basic dXNlcjpwYXNz==')).toEqual({ + Authorization: 'Basic dXNlcjpwYXNz==', + }); + }); + + it('strips surrounding quotes from values in key=value format', () => { + expect( + parseOtlpHeaders('Authorization="Bearer token",x-key=\'val\''), + ).toEqual({ + Authorization: 'Bearer token', + 'x-key': 'val', + }); + }); + + it('trims whitespace around keys and values', () => { + expect( + parseOtlpHeaders(' Authorization = Bearer token , x-key = val '), + ).toEqual({ + Authorization: 'Bearer token', + 'x-key': 'val', + }); + }); + + it('returns undefined for invalid key=value strings', () => { + expect(parseOtlpHeaders('invalid-no-equals')).toBeUndefined(); + expect(parseOtlpHeaders('=value_without_key')).toBeUndefined(); + }); + + it('rejects header names with invalid RFC 7230 characters', () => { + expect(parseOtlpHeaders('bad name=value')).toBeUndefined(); + expect(parseOtlpHeaders('{"bad name": "value"}')).toBeUndefined(); + expect(parseOtlpHeaders('bad:name=value')).toBeUndefined(); + }); + + it('rejects header values with control characters (CRLF injection)', () => { + expect(parseOtlpHeaders('x-key=val\r\nInjected: true')).toBeUndefined(); + expect(parseOtlpHeaders('{"x-key": "val\\u0000bad"}')).toBeUndefined(); + }); + + it('skips invalid pairs in key=value format and keeps valid ones', () => { + expect( + parseOtlpHeaders('Authorization=Bearer token,invalid,=nokey,x-ok=yes'), + ).toEqual({ + Authorization: 'Bearer token', + 'x-ok': 'yes', + }); + }); + }); + describe('resolveTelemetrySettings', () => { it('falls back to settings when no argv/env provided', async () => { const settings = { @@ -179,5 +276,79 @@ describe('telemetry/config helpers', () => { /Invalid telemetry target/i, ); }); + + it('resolves otlpHeaders from settings', async () => { + const settings = { + otlpHeaders: { Authorization: 'Bearer settings-token' }, + }; + const resolved = await resolveTelemetrySettings({ settings }); + expect(resolved.otlpHeaders).toEqual({ + Authorization: 'Bearer settings-token', + }); + }); + + it('resolves otlpHeaders from GEMINI_TELEMETRY_OTLP_HEADERS and OTEL_EXPORTER_OTLP_HEADERS', async () => { + const resolvedOtel = await resolveTelemetrySettings({ + env: { OTEL_EXPORTER_OTLP_HEADERS: 'x-otel=1,Authorization=Bearer a' }, + }); + expect(resolvedOtel.otlpHeaders).toEqual({ + 'x-otel': '1', + Authorization: 'Bearer a', + }); + + const resolvedGemini = await resolveTelemetrySettings({ + env: { + OTEL_EXPORTER_OTLP_HEADERS: 'x-otel=1,Authorization=Bearer a', + GEMINI_TELEMETRY_OTLP_HEADERS: '{"Authorization":"Bearer b"}', + }, + }); + expect(resolvedGemini.otlpHeaders).toEqual({ + 'x-otel': '1', + Authorization: 'Bearer b', + }); + }); + + it('merges otlpHeaders across settings, env, and argv with case-insensitive deduplication', async () => { + const settings = { + otlpHeaders: { + Authorization: 'Bearer settings', + 'X-Settings': 'from-settings', + }, + }; + const env = { + GEMINI_TELEMETRY_OTLP_HEADERS: + 'authorization=Bearer env,X-Env=from-env', + }; + const argv = { + telemetryOtlpHeaders: 'X-Argv=from-argv', + }; + const resolved = await resolveTelemetrySettings({ argv, env, settings }); + expect(resolved.otlpHeaders).toEqual({ + 'X-Settings': 'from-settings', + authorization: 'Bearer env', + 'X-Env': 'from-env', + 'X-Argv': 'from-argv', + }); + }); + + it('throws FatalConfigError on invalid otlpHeaders in env or argv', async () => { + await expect( + resolveTelemetrySettings({ + env: { GEMINI_TELEMETRY_OTLP_HEADERS: 'invalid-headers' }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers/i); + + await expect( + resolveTelemetrySettings({ + env: { OTEL_EXPORTER_OTLP_HEADERS: '{bad json}' }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers/i); + + await expect( + resolveTelemetrySettings({ + argv: { telemetryOtlpHeaders: 'invalid-headers' }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers/i); + }); }); }); diff --git a/packages/core/src/telemetry/config.ts b/packages/core/src/telemetry/config.ts index 9fd4bacfc3b..3ecb91ca300 100644 --- a/packages/core/src/telemetry/config.ts +++ b/packages/core/src/telemetry/config.ts @@ -39,10 +39,130 @@ export interface TelemetryArgOverrides { telemetryTarget?: string | TelemetryTarget; telemetryOtlpEndpoint?: string; telemetryOtlpProtocol?: string; + telemetryOtlpHeaders?: string | Record; telemetryLogPrompts?: boolean; telemetryOutfile?: string; } +const HEADER_NAME_REGEX = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; + +function isValidHeaderName(name: string): boolean { + return HEADER_NAME_REGEX.test(name); +} + +function isValidHeaderValue(value: string): boolean { + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if ((code < 0x20 && code !== 0x09) || code === 0x7f) { + return false; + } + } + return true; +} + +function stripQuotes(value: string): string { + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + return value.slice(1, -1); + } + return value; +} + +/** + * Parse OTLP headers from a string. + * Supports JSON object format (e.g., '{"Authorization":"Bearer token"}') or + * key=value pairs separated by commas or semicolons (e.g., 'Authorization=Bearer token,x-api-key=abc123'). + */ +export function parseOtlpHeaders( + value: string | undefined, +): Record | undefined { + if (!value || value.trim() === '') return undefined; + + const trimmed = value.trim(); + + // If the string starts with '{', treat it strictly as JSON and do not fall through to key=value parsing. + if (trimmed.startsWith('{')) { + try { + const parsed: unknown = JSON.parse(trimmed); + if ( + typeof parsed === 'object' && + parsed !== null && + !Array.isArray(parsed) + ) { + const headers: Record = {}; + for (const [k, v] of Object.entries(parsed)) { + const trimmedKey = k.trim(); + if ( + typeof v !== 'string' || + !isValidHeaderName(trimmedKey) || + !isValidHeaderValue(v) + ) { + return undefined; + } + headers[trimmedKey] = v; + } + return Object.keys(headers).length > 0 ? headers : undefined; + } + return undefined; + } catch { + return undefined; + } + } + + // Reject other JSON literals like arrays + if (trimmed.startsWith('[')) { + return undefined; + } + + // Parse as key=value pairs separated by commas or semicolons + const headers: Record = {}; + const pairs = trimmed.split(/[,;]/); + for (const pair of pairs) { + const trimmedPair = pair.trim(); + if (!trimmedPair) continue; + const eqIndex = trimmedPair.indexOf('='); + if (eqIndex === -1) continue; + const key = trimmedPair.slice(0, eqIndex).trim(); + const val = stripQuotes(trimmedPair.slice(eqIndex + 1).trim()); + if (key && isValidHeaderName(key) && isValidHeaderValue(val)) { + headers[key] = val; + } + } + + return Object.keys(headers).length > 0 ? headers : undefined; +} + +/** + * Merge header objects from lowest to highest precedence, deduplicating keys case-insensitively. + */ +function mergeHeaders( + ...sources: Array | undefined> +): Record | undefined { + const definedSources = sources.filter( + (s): s is Record => s !== undefined, + ); + if (definedSources.length === 0) { + return undefined; + } + + const merged: Record = {}; + const lowerKeyMap = new Map(); + for (const source of definedSources) { + for (const [key, value] of Object.entries(source)) { + const lowerKey = key.toLowerCase(); + const existingKey = lowerKeyMap.get(lowerKey); + if (existingKey !== undefined && existingKey !== key) { + delete merged[existingKey]; + } + merged[key] = value; + lowerKeyMap.set(lowerKey, key); + } + } + return merged; +} + /** * Build TelemetrySettings by resolving from argv (highest), env, then settings. */ @@ -112,12 +232,60 @@ export async function resolveTelemetrySettings(options: { parseBooleanEnvFlag(env['GEMINI_TELEMETRY_USE_COLLECTOR']) ?? settings.useCollector; + // Resolve OTLP headers: merge settings (lowest), OTEL_EXPORTER_OTLP_HEADERS, + // GEMINI_TELEMETRY_OTLP_HEADERS, and argv (highest). + const rawOtelEnvHeaders = env['OTEL_EXPORTER_OTLP_HEADERS']; + let otelEnvHeaders: Record | undefined; + if (rawOtelEnvHeaders !== undefined && rawOtelEnvHeaders.trim() !== '') { + otelEnvHeaders = parseOtlpHeaders(rawOtelEnvHeaders); + if (otelEnvHeaders === undefined) { + throw new FatalConfigError( + `Invalid telemetry OTLP headers: ${rawOtelEnvHeaders}. Expected JSON object or key=value pairs`, + ); + } + } + + const rawGeminiEnvHeaders = env['GEMINI_TELEMETRY_OTLP_HEADERS']; + let geminiEnvHeaders: Record | undefined; + if (rawGeminiEnvHeaders !== undefined && rawGeminiEnvHeaders.trim() !== '') { + geminiEnvHeaders = parseOtlpHeaders(rawGeminiEnvHeaders); + if (geminiEnvHeaders === undefined) { + throw new FatalConfigError( + `Invalid telemetry OTLP headers: ${rawGeminiEnvHeaders}. Expected JSON object or key=value pairs`, + ); + } + } + + let argvHeaders: Record | undefined; + if (argv.telemetryOtlpHeaders !== undefined) { + if (typeof argv.telemetryOtlpHeaders === 'string') { + if (argv.telemetryOtlpHeaders.trim() !== '') { + argvHeaders = parseOtlpHeaders(argv.telemetryOtlpHeaders); + if (argvHeaders === undefined) { + throw new FatalConfigError( + `Invalid telemetry OTLP headers: ${argv.telemetryOtlpHeaders}. Expected JSON object or key=value pairs`, + ); + } + } + } else { + argvHeaders = argv.telemetryOtlpHeaders; + } + } + + const otlpHeaders = mergeHeaders( + settings.otlpHeaders, + otelEnvHeaders, + geminiEnvHeaders, + argvHeaders, + ); + return { enabled, traces, target, otlpEndpoint, otlpProtocol, + otlpHeaders, logPrompts, outfile, useCollector, diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index 83e5517882f..d1ca14d276b 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -23,6 +23,7 @@ export { resolveTelemetrySettings, parseBooleanEnvFlag, parseTelemetryTargetValue, + parseOtlpHeaders, } from './config.js'; export { GcpTraceExporter, diff --git a/packages/core/src/telemetry/sdk.test.ts b/packages/core/src/telemetry/sdk.test.ts index 9636212e2c9..16c50972ae2 100644 --- a/packages/core/src/telemetry/sdk.test.ts +++ b/packages/core/src/telemetry/sdk.test.ts @@ -17,6 +17,7 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; import { OTLPTraceExporter as OTLPTraceExporterHttp } from '@opentelemetry/exporter-trace-otlp-http'; import { OTLPLogExporter as OTLPLogExporterHttp } from '@opentelemetry/exporter-logs-otlp-http'; import { OTLPMetricExporter as OTLPMetricExporterHttp } from '@opentelemetry/exporter-metrics-otlp-http'; +import { Metadata } from '@grpc/grpc-js'; import { ConsoleSpanExporter } from '@opentelemetry/sdk-trace-node'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { GoogleAuth, type JWTInput } from 'google-auth-library'; @@ -68,6 +69,7 @@ describe('Telemetry SDK', () => { getTelemetryEnabled: () => true, getTelemetryOtlpEndpoint: () => 'http://localhost:4317', getTelemetryOtlpProtocol: () => 'grpc', + getTelemetryOtlpHeaders: () => ({}), getTelemetryTarget: () => 'local', getTelemetryUseCollector: () => false, getTelemetryOutfile: () => undefined, @@ -124,6 +126,71 @@ describe('Telemetry SDK', () => { expect(NodeSDK.prototype.start).toHaveBeenCalled(); }); + it('should pass custom headers to HTTP exporters when otlpHeaders is configured', async () => { + const customHeaders = { + Authorization: 'Bearer secret-token', + 'x-api-key': 'my-api-key', + }; + vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('http'); + vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( + 'http://localhost:4318', + ); + vi.spyOn(mockConfig, 'getTelemetryOtlpHeaders').mockReturnValue( + customHeaders, + ); + + await initializeTelemetry(mockConfig); + + expect(OTLPTraceExporterHttp).toHaveBeenCalledWith({ + url: 'http://localhost:4318/v1/traces', + headers: customHeaders, + }); + expect(OTLPLogExporterHttp).toHaveBeenCalledWith({ + url: 'http://localhost:4318/v1/logs', + headers: customHeaders, + }); + expect(OTLPMetricExporterHttp).toHaveBeenCalledWith({ + url: 'http://localhost:4318/v1/metrics', + headers: customHeaders, + }); + }); + + it('should pass custom metadata to gRPC exporters when otlpHeaders is configured', async () => { + const customHeaders = { + Authorization: 'Bearer secret-token', + 'x-api-key': 'my-api-key', + }; + vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('grpc'); + vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( + 'http://localhost:4317', + ); + vi.spyOn(mockConfig, 'getTelemetryOtlpHeaders').mockReturnValue( + customHeaders, + ); + + await initializeTelemetry(mockConfig); + + for (const ExporterMock of [ + OTLPTraceExporter, + OTLPLogExporter, + OTLPMetricExporter, + ]) { + expect(ExporterMock).toHaveBeenCalledTimes(1); + const callArg = vi.mocked(ExporterMock).mock.calls[0]?.[0] as { + url?: string; + compression?: string; + metadata?: Metadata; + }; + expect(callArg?.url).toBe('http://localhost:4317'); + expect(callArg?.compression).toBe('gzip'); + expect(callArg?.metadata).toBeInstanceOf(Metadata); + expect(callArg?.metadata?.get('authorization')).toEqual([ + 'Bearer secret-token', + ]); + expect(callArg?.metadata?.get('x-api-key')).toEqual(['my-api-key']); + } + }); + it('should parse gRPC endpoint correctly', async () => { vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( 'https://my-collector.com', diff --git a/packages/core/src/telemetry/sdk.ts b/packages/core/src/telemetry/sdk.ts index ac90bf86ad8..f74db3a0c59 100644 --- a/packages/core/src/telemetry/sdk.ts +++ b/packages/core/src/telemetry/sdk.ts @@ -19,6 +19,7 @@ import { OTLPTraceExporter as OTLPTraceExporterHttp } from '@opentelemetry/expor import { OTLPLogExporter as OTLPLogExporterHttp } from '@opentelemetry/exporter-logs-otlp-http'; import { OTLPMetricExporter as OTLPMetricExporterHttp } from '@opentelemetry/exporter-metrics-otlp-http'; import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base'; +import { Metadata } from '@grpc/grpc-js'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import { resourceFromAttributes } from '@opentelemetry/resources'; @@ -279,6 +280,9 @@ export async function initializeTelemetry( exportIntervalMillis: 30000, }); } else if (useOtlp) { + const otlpHeaders = config.getTelemetryOtlpHeaders?.() ?? {}; + const hasHeaders = Object.keys(otlpHeaders).length > 0; + if (otlpProtocol === 'http') { const buildUrl = (path: string) => { const url = new URL(parsedEndpoint); @@ -288,30 +292,43 @@ export async function initializeTelemetry( }; spanExporter = new OTLPTraceExporterHttp({ url: buildUrl('v1/traces'), + headers: hasHeaders ? otlpHeaders : undefined, }); logExporter = new OTLPLogExporterHttp({ url: buildUrl('v1/logs'), + headers: hasHeaders ? otlpHeaders : undefined, }); metricReader = new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporterHttp({ url: buildUrl('v1/metrics'), + headers: hasHeaders ? otlpHeaders : undefined, }), exportIntervalMillis: 10000, }); } else { // grpc + let metadata: Metadata | undefined; + if (hasHeaders) { + metadata = new Metadata(); + for (const [key, value] of Object.entries(otlpHeaders)) { + metadata.set(key, value); + } + } spanExporter = new OTLPTraceExporter({ url: parsedEndpoint, compression: CompressionAlgorithm.GZIP, + ...(metadata && { metadata }), }); logExporter = new OTLPLogExporter({ url: parsedEndpoint, compression: CompressionAlgorithm.GZIP, + ...(metadata && { metadata }), }); metricReader = new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter({ url: parsedEndpoint, compression: CompressionAlgorithm.GZIP, + ...(metadata && { metadata }), }), exportIntervalMillis: 10000, }); diff --git a/schemas/settings.schema.json b/schemas/settings.schema.json index c912b86ae71..992d2eaed35 100644 --- a/schemas/settings.schema.json +++ b/schemas/settings.schema.json @@ -4270,6 +4270,13 @@ "description": "Protocol for OTLP exporters.", "enum": ["grpc", "http"] }, + "otlpHeaders": { + "type": "object", + "description": "Custom headers to send with OTLP exporter requests.", + "additionalProperties": { + "type": "string" + } + }, "traces": { "type": "boolean", "description": "Whether detailed traces with large attributes are captured." From 0320fff7e5c2120cce7f08ddbc1ff5741555f43c Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Mon, 5 Oct 2026 17:39:17 +0000 Subject: [PATCH 2/6] fix(telemetry): address PR review feedback on OTLP headers and tests --- docs/reference/configuration.md | 2 +- packages/cli/src/config/settings.test.ts | 4 ++-- packages/core/src/telemetry/config.test.ts | 10 ++++---- packages/core/src/telemetry/config.ts | 6 ++--- packages/core/src/telemetry/sdk.test.ts | 28 ++++++++++++++++++++++ packages/core/src/telemetry/sdk.ts | 16 +++++++++++-- 6 files changed, 54 insertions(+), 12 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0ccca52328e..4925aeaa705 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -2771,7 +2771,7 @@ the `advanced.excludedEnvVars` setting in your `settings.json` file. - Overrides the `telemetry.otlpProtocol` setting. - **`GEMINI_TELEMETRY_OTLP_HEADERS`**: - Sets custom headers for OTLP exporter requests (as a JSON object string or - comma/semicolon-separated `key=value` pairs; also falls back to + comma-separated `key=value` pairs; also falls back to `OTEL_EXPORTER_OTLP_HEADERS`). - Merges with and overrides headers in the `telemetry.otlpHeaders` setting. - **`GEMINI_TELEMETRY_LOG_PROMPTS`**: diff --git a/packages/cli/src/config/settings.test.ts b/packages/cli/src/config/settings.test.ts index 545a77363b0..89f0028d8a7 100644 --- a/packages/cli/src/config/settings.test.ts +++ b/packages/cli/src/config/settings.test.ts @@ -904,7 +904,7 @@ describe('Settings Loading and Merging', () => { }); it('should load telemetry.otlpHeaders and resolve environment variables', () => { - process.env['TEST_OTLP_TOKEN'] = 'secret-token-value'; + vi.stubEnv('TEST_OTLP_TOKEN', 'secret-token-value'); try { (mockFsExistsSync as Mock).mockImplementation( (p: fs.PathLike) => @@ -932,7 +932,7 @@ describe('Settings Loading and Merging', () => { 'x-custom': 'static-val', }); } finally { - delete process.env['TEST_OTLP_TOKEN']; + vi.unstubAllEnvs(); } }); diff --git a/packages/core/src/telemetry/config.test.ts b/packages/core/src/telemetry/config.test.ts index f44933376af..4282d45228e 100644 --- a/packages/core/src/telemetry/config.test.ts +++ b/packages/core/src/telemetry/config.test.ts @@ -91,12 +91,14 @@ describe('telemetry/config helpers', () => { }); }); - it('parses semicolon-separated key=value pairs', () => { + it('preserves semicolons inside header values in key=value format', () => { expect( - parseOtlpHeaders('Authorization=Bearer token123;x-api-key=abc'), + parseOtlpHeaders( + 'Authorization=Bearer token123;tenant=abc,x-api-key=xyz', + ), ).toEqual({ - Authorization: 'Bearer token123', - 'x-api-key': 'abc', + Authorization: 'Bearer token123;tenant=abc', + 'x-api-key': 'xyz', }); }); diff --git a/packages/core/src/telemetry/config.ts b/packages/core/src/telemetry/config.ts index 3ecb91ca300..69168c091b5 100644 --- a/packages/core/src/telemetry/config.ts +++ b/packages/core/src/telemetry/config.ts @@ -73,7 +73,7 @@ function stripQuotes(value: string): string { /** * Parse OTLP headers from a string. * Supports JSON object format (e.g., '{"Authorization":"Bearer token"}') or - * key=value pairs separated by commas or semicolons (e.g., 'Authorization=Bearer token,x-api-key=abc123'). + * key=value pairs separated by commas (e.g., 'Authorization=Bearer token,x-api-key=abc123'). */ export function parseOtlpHeaders( value: string | undefined, @@ -116,9 +116,9 @@ export function parseOtlpHeaders( return undefined; } - // Parse as key=value pairs separated by commas or semicolons + // Parse as key=value pairs separated by commas const headers: Record = {}; - const pairs = trimmed.split(/[,;]/); + const pairs = trimmed.split(','); for (const pair of pairs) { const trimmedPair = pair.trim(); if (!trimmedPair) continue; diff --git a/packages/core/src/telemetry/sdk.test.ts b/packages/core/src/telemetry/sdk.test.ts index 16c50972ae2..ab250416706 100644 --- a/packages/core/src/telemetry/sdk.test.ts +++ b/packages/core/src/telemetry/sdk.test.ts @@ -191,6 +191,34 @@ describe('Telemetry SDK', () => { } }); + it('should skip invalid gRPC metadata keys and log a warning without crashing', async () => { + const customHeaders = { + bad$key: 'invalid-for-grpc', + 'x-valid-key': 'valid-value', + }; + vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('grpc'); + vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( + 'http://localhost:4317', + ); + vi.spyOn(mockConfig, 'getTelemetryOtlpHeaders').mockReturnValue( + customHeaders, + ); + + await initializeTelemetry(mockConfig); + + expect(debugLogger.warn).toHaveBeenCalledWith( + expect.stringContaining( + '[Telemetry] Skipping invalid gRPC metadata key "bad$key":', + ), + expect.any(Error), + ); + const callArg = vi.mocked(OTLPTraceExporter).mock.calls[0]?.[0] as { + metadata?: Metadata; + }; + expect(callArg?.metadata).toBeInstanceOf(Metadata); + expect(callArg?.metadata?.get('x-valid-key')).toEqual(['valid-value']); + }); + it('should parse gRPC endpoint correctly', async () => { vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( 'https://my-collector.com', diff --git a/packages/core/src/telemetry/sdk.ts b/packages/core/src/telemetry/sdk.ts index f74db3a0c59..d3f3495f563 100644 --- a/packages/core/src/telemetry/sdk.ts +++ b/packages/core/src/telemetry/sdk.ts @@ -309,9 +309,21 @@ export async function initializeTelemetry( // grpc let metadata: Metadata | undefined; if (hasHeaders) { - metadata = new Metadata(); + const candidateMetadata = new Metadata(); + let validCount = 0; for (const [key, value] of Object.entries(otlpHeaders)) { - metadata.set(key, value); + try { + candidateMetadata.set(key, value); + validCount++; + } catch (error) { + debugLogger.warn( + `[Telemetry] Skipping invalid gRPC metadata key "${key}":`, + error, + ); + } + } + if (validCount > 0) { + metadata = candidateMetadata; } } spanExporter = new OTLPTraceExporter({ From 77553f65da6bb7b37e501747ed87206a18676528 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Mon, 5 Oct 2026 18:14:05 +0000 Subject: [PATCH 3/6] fix(telemetry): sanitize telemetry env keys and validate object otlpHeaders --- docs/reference/configuration.md | 3 +- packages/cli/src/config/settings.test.ts | 55 ++++----- packages/core/src/telemetry/config.test.ts | 12 ++ packages/core/src/telemetry/config.ts | 126 +++++++++++++++------ 4 files changed, 129 insertions(+), 67 deletions(-) diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 4925aeaa705..68db15f69f7 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -2551,8 +2551,7 @@ see [Telemetry](../cli/telemetry.md). - **`otlpProtocol`** (string): The protocol for the OTLP Exporter (`grpc` or `http`). - **`otlpHeaders`** (object): Custom headers to send with OTLP Exporter - requests (supports environment variable references like `$VAR_NAME` or - `${VAR_NAME}`). + requests. - **`logPrompts`** (boolean): Whether or not to include the content of user prompts in the logs. - **`outfile`** (string): The file to write telemetry to when `target` is diff --git a/packages/cli/src/config/settings.test.ts b/packages/cli/src/config/settings.test.ts index 89f0028d8a7..beb6cab03ac 100644 --- a/packages/cli/src/config/settings.test.ts +++ b/packages/cli/src/config/settings.test.ts @@ -903,37 +903,32 @@ describe('Settings Loading and Merging', () => { expect(settings.merged.telemetry?.enabled).toBe(false); }); - it('should load telemetry.otlpHeaders and resolve environment variables', () => { - vi.stubEnv('TEST_OTLP_TOKEN', 'secret-token-value'); - try { - (mockFsExistsSync as Mock).mockImplementation( - (p: fs.PathLike) => - normalizePath(p) === normalizePath(USER_SETTINGS_PATH), - ); - const userSettingsContent = { - telemetry: { - enabled: true, - otlpHeaders: { - Authorization: 'Bearer ${TEST_OTLP_TOKEN}', - 'x-custom': 'static-val', - }, - }, - }; - (fs.readFileSync as Mock).mockImplementation( - (p: fs.PathOrFileDescriptor) => { - if (normalizePath(p) === normalizePath(USER_SETTINGS_PATH)) - return JSON.stringify(userSettingsContent); - return '{}'; + it('should load telemetry.otlpHeaders from settings', () => { + (mockFsExistsSync as Mock).mockImplementation( + (p: fs.PathLike) => + normalizePath(p) === normalizePath(USER_SETTINGS_PATH), + ); + const userSettingsContent = { + telemetry: { + enabled: true, + otlpHeaders: { + Authorization: 'Bearer test-token-value', + 'x-custom': 'static-val', }, - ); - const settings = loadSettings(MOCK_WORKSPACE_DIR); - expect(settings.merged.telemetry?.otlpHeaders).toEqual({ - Authorization: 'Bearer secret-token-value', - 'x-custom': 'static-val', - }); - } finally { - vi.unstubAllEnvs(); - } + }, + }; + (fs.readFileSync as Mock).mockImplementation( + (p: fs.PathOrFileDescriptor) => { + if (normalizePath(p) === normalizePath(USER_SETTINGS_PATH)) + return JSON.stringify(userSettingsContent); + return '{}'; + }, + ); + const settings = loadSettings(MOCK_WORKSPACE_DIR); + expect(settings.merged.telemetry?.otlpHeaders).toEqual({ + Authorization: 'Bearer test-token-value', + 'x-custom': 'static-val', + }); }); it('should have telemetry as undefined if not in any settings file', () => { diff --git a/packages/core/src/telemetry/config.test.ts b/packages/core/src/telemetry/config.test.ts index 4282d45228e..92d70a4b26c 100644 --- a/packages/core/src/telemetry/config.test.ts +++ b/packages/core/src/telemetry/config.test.ts @@ -351,6 +351,18 @@ describe('telemetry/config helpers', () => { argv: { telemetryOtlpHeaders: 'invalid-headers' }, }), ).rejects.toThrow(/Invalid telemetry OTLP headers/i); + + await expect( + resolveTelemetrySettings({ + settings: { otlpHeaders: { 'bad name': 'value' } }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in settings/i); + + await expect( + resolveTelemetrySettings({ + argv: { telemetryOtlpHeaders: { 'x-key': 'bad\r\nvalue' } }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in argv/i); }); }); }); diff --git a/packages/core/src/telemetry/config.ts b/packages/core/src/telemetry/config.ts index 69168c091b5..335359fafdf 100644 --- a/packages/core/src/telemetry/config.ts +++ b/packages/core/src/telemetry/config.ts @@ -70,6 +70,56 @@ function stripQuotes(value: string): string { return value; } +const ALLOWED_TELEMETRY_ENV_KEYS = new Set([ + 'GEMINI_TELEMETRY_ENABLED', + 'GEMINI_TELEMETRY_TRACES_ENABLED', + 'GEMINI_TELEMETRY_TARGET', + 'GEMINI_TELEMETRY_OTLP_ENDPOINT', + 'OTEL_EXPORTER_OTLP_ENDPOINT', + 'GEMINI_TELEMETRY_OTLP_PROTOCOL', + 'GEMINI_TELEMETRY_OTLP_HEADERS', + 'OTEL_EXPORTER_OTLP_HEADERS', + 'GEMINI_TELEMETRY_LOG_PROMPTS', + 'GEMINI_TELEMETRY_OUTFILE', + 'GEMINI_TELEMETRY_USE_COLLECTOR', + 'GEMINI_TELEMETRY_USE_CLI_AUTH', +]); + +function setHeaderCaseInsensitive( + headers: Record, + lowerKeyMap: Map, + key: string, + value: string, +): void { + const lowerKey = key.toLowerCase(); + const existingKey = lowerKeyMap.get(lowerKey); + if (existingKey !== undefined && existingKey !== key) { + delete headers[existingKey]; + } + headers[key] = value; + lowerKeyMap.set(lowerKey, key); +} + +function validateHeadersObject( + headers: object, +): Record | undefined { + const validated: Record = {}; + const lowerKeyMap = new Map(); + for (const [k, v] of Object.entries(headers)) { + const trimmedKey = k.trim(); + if ( + typeof v !== 'string' || + !trimmedKey || + !isValidHeaderName(trimmedKey) || + !isValidHeaderValue(v) + ) { + return undefined; + } + setHeaderCaseInsensitive(validated, lowerKeyMap, trimmedKey, v); + } + return validated; +} + /** * Parse OTLP headers from a string. * Supports JSON object format (e.g., '{"Authorization":"Bearer token"}') or @@ -91,19 +141,11 @@ export function parseOtlpHeaders( parsed !== null && !Array.isArray(parsed) ) { - const headers: Record = {}; - for (const [k, v] of Object.entries(parsed)) { - const trimmedKey = k.trim(); - if ( - typeof v !== 'string' || - !isValidHeaderName(trimmedKey) || - !isValidHeaderValue(v) - ) { - return undefined; - } - headers[trimmedKey] = v; + const validated = validateHeadersObject(parsed); + if (!validated || Object.keys(validated).length === 0) { + return undefined; } - return Object.keys(headers).length > 0 ? headers : undefined; + return validated; } return undefined; } catch { @@ -118,6 +160,7 @@ export function parseOtlpHeaders( // Parse as key=value pairs separated by commas const headers: Record = {}; + const lowerKeyMap = new Map(); const pairs = trimmed.split(','); for (const pair of pairs) { const trimmedPair = pair.trim(); @@ -127,7 +170,7 @@ export function parseOtlpHeaders( const key = trimmedPair.slice(0, eqIndex).trim(); const val = stripQuotes(trimmedPair.slice(eqIndex + 1).trim()); if (key && isValidHeaderName(key) && isValidHeaderValue(val)) { - headers[key] = val; + setHeaderCaseInsensitive(headers, lowerKeyMap, key, val); } } @@ -151,13 +194,7 @@ function mergeHeaders( const lowerKeyMap = new Map(); for (const source of definedSources) { for (const [key, value] of Object.entries(source)) { - const lowerKey = key.toLowerCase(); - const existingKey = lowerKeyMap.get(lowerKey); - if (existingKey !== undefined && existingKey !== key) { - delete merged[existingKey]; - } - merged[key] = value; - lowerKeyMap.set(lowerKey, key); + setHeaderCaseInsensitive(merged, lowerKeyMap, key, value); } } return merged; @@ -172,21 +209,25 @@ export async function resolveTelemetrySettings(options: { settings?: TelemetrySettings; }): Promise { const argv = options.argv ?? {}; - const env = options.env ?? {}; + const sanitizedEnv = Object.fromEntries( + Object.entries(options.env ?? {}).filter(([key]) => + ALLOWED_TELEMETRY_ENV_KEYS.has(key), + ), + ); const settings = options.settings ?? {}; const enabled = argv.telemetry ?? - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_ENABLED']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_ENABLED']) ?? settings.enabled; const traces = - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_TRACES_ENABLED']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_TRACES_ENABLED']) ?? settings.traces; const rawTarget = argv.telemetryTarget ?? - env['GEMINI_TELEMETRY_TARGET'] ?? + sanitizedEnv['GEMINI_TELEMETRY_TARGET'] ?? (settings.target as string | TelemetryTarget | undefined); const target = parseTelemetryTargetValue(rawTarget); if (rawTarget !== undefined && target === undefined) { @@ -199,13 +240,13 @@ export async function resolveTelemetrySettings(options: { const otlpEndpoint = argv.telemetryOtlpEndpoint ?? - env['GEMINI_TELEMETRY_OTLP_ENDPOINT'] ?? - env['OTEL_EXPORTER_OTLP_ENDPOINT'] ?? + sanitizedEnv['GEMINI_TELEMETRY_OTLP_ENDPOINT'] ?? + sanitizedEnv['OTEL_EXPORTER_OTLP_ENDPOINT'] ?? settings.otlpEndpoint; const rawProtocol = argv.telemetryOtlpProtocol ?? - env['GEMINI_TELEMETRY_OTLP_PROTOCOL'] ?? + sanitizedEnv['GEMINI_TELEMETRY_OTLP_PROTOCOL'] ?? settings.otlpProtocol; const otlpProtocol = (['grpc', 'http'] as const).find( (p) => p === rawProtocol, @@ -220,21 +261,31 @@ export async function resolveTelemetrySettings(options: { const logPrompts = argv.telemetryLogPrompts ?? - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_LOG_PROMPTS']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_LOG_PROMPTS']) ?? settings.logPrompts; const outfile = argv.telemetryOutfile ?? - env['GEMINI_TELEMETRY_OUTFILE'] ?? + sanitizedEnv['GEMINI_TELEMETRY_OUTFILE'] ?? settings.outfile; const useCollector = - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_USE_COLLECTOR']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_USE_COLLECTOR']) ?? settings.useCollector; // Resolve OTLP headers: merge settings (lowest), OTEL_EXPORTER_OTLP_HEADERS, // GEMINI_TELEMETRY_OTLP_HEADERS, and argv (highest). - const rawOtelEnvHeaders = env['OTEL_EXPORTER_OTLP_HEADERS']; + let settingsHeaders: Record | undefined; + if (settings.otlpHeaders !== undefined) { + settingsHeaders = validateHeadersObject(settings.otlpHeaders); + if (settingsHeaders === undefined) { + throw new FatalConfigError( + 'Invalid telemetry OTLP headers in settings: header names and values must be valid HTTP header tokens.', + ); + } + } + + const rawOtelEnvHeaders = sanitizedEnv['OTEL_EXPORTER_OTLP_HEADERS']; let otelEnvHeaders: Record | undefined; if (rawOtelEnvHeaders !== undefined && rawOtelEnvHeaders.trim() !== '') { otelEnvHeaders = parseOtlpHeaders(rawOtelEnvHeaders); @@ -245,7 +296,7 @@ export async function resolveTelemetrySettings(options: { } } - const rawGeminiEnvHeaders = env['GEMINI_TELEMETRY_OTLP_HEADERS']; + const rawGeminiEnvHeaders = sanitizedEnv['GEMINI_TELEMETRY_OTLP_HEADERS']; let geminiEnvHeaders: Record | undefined; if (rawGeminiEnvHeaders !== undefined && rawGeminiEnvHeaders.trim() !== '') { geminiEnvHeaders = parseOtlpHeaders(rawGeminiEnvHeaders); @@ -268,12 +319,17 @@ export async function resolveTelemetrySettings(options: { } } } else { - argvHeaders = argv.telemetryOtlpHeaders; + argvHeaders = validateHeadersObject(argv.telemetryOtlpHeaders); + if (argvHeaders === undefined) { + throw new FatalConfigError( + 'Invalid telemetry OTLP headers in argv: header names and values must be valid HTTP header tokens.', + ); + } } } const otlpHeaders = mergeHeaders( - settings.otlpHeaders, + settingsHeaders, otelEnvHeaders, geminiEnvHeaders, argvHeaders, @@ -290,7 +346,7 @@ export async function resolveTelemetrySettings(options: { outfile, useCollector, useCliAuth: - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_USE_CLI_AUTH']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_USE_CLI_AUTH']) ?? settings.useCliAuth, }; } From a7893f28bfb217de1c0b10b6c456710f1d0b7694 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Mon, 5 Oct 2026 18:38:09 +0000 Subject: [PATCH 4/6] fix(telemetry): redact raw OTLP header values from errors and guard non-object headers --- packages/core/src/telemetry/config.test.ts | 40 ++++++++++++++++++---- packages/core/src/telemetry/config.ts | 30 ++++++++-------- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/packages/core/src/telemetry/config.test.ts b/packages/core/src/telemetry/config.test.ts index 92d70a4b26c..43afe6b0af3 100644 --- a/packages/core/src/telemetry/config.test.ts +++ b/packages/core/src/telemetry/config.test.ts @@ -333,24 +333,34 @@ describe('telemetry/config helpers', () => { }); }); - it('throws FatalConfigError on invalid otlpHeaders in env or argv', async () => { + it('throws FatalConfigError on invalid otlpHeaders without leaking raw secret values', async () => { await expect( resolveTelemetrySettings({ - env: { GEMINI_TELEMETRY_OTLP_HEADERS: 'invalid-headers' }, + env: { + GEMINI_TELEMETRY_OTLP_HEADERS: '{"Authorization":"secret-token-1"', + }, }), - ).rejects.toThrow(/Invalid telemetry OTLP headers/i); + ).rejects.toThrow( + 'Invalid telemetry OTLP headers in GEMINI_TELEMETRY_OTLP_HEADERS. Expected JSON object or key=value pairs', + ); await expect( resolveTelemetrySettings({ - env: { OTEL_EXPORTER_OTLP_HEADERS: '{bad json}' }, + env: { + OTEL_EXPORTER_OTLP_HEADERS: '{"Authorization":"secret-token-2"', + }, }), - ).rejects.toThrow(/Invalid telemetry OTLP headers/i); + ).rejects.toThrow( + 'Invalid telemetry OTLP headers in OTEL_EXPORTER_OTLP_HEADERS. Expected JSON object or key=value pairs', + ); await expect( resolveTelemetrySettings({ - argv: { telemetryOtlpHeaders: 'invalid-headers' }, + argv: { telemetryOtlpHeaders: '{"Authorization":"secret-token-3"' }, }), - ).rejects.toThrow(/Invalid telemetry OTLP headers/i); + ).rejects.toThrow( + 'Invalid telemetry OTLP headers in argv. Expected JSON object or key=value pairs', + ); await expect( resolveTelemetrySettings({ @@ -358,11 +368,27 @@ describe('telemetry/config helpers', () => { }), ).rejects.toThrow(/Invalid telemetry OTLP headers in settings/i); + await expect( + resolveTelemetrySettings({ + settings: { + otlpHeaders: null as unknown as Record, + }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in settings/i); + await expect( resolveTelemetrySettings({ argv: { telemetryOtlpHeaders: { 'x-key': 'bad\r\nvalue' } }, }), ).rejects.toThrow(/Invalid telemetry OTLP headers in argv/i); + + await expect( + resolveTelemetrySettings({ + argv: { + telemetryOtlpHeaders: ['a=b'] as unknown as Record, + }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in argv/i); }); }); }); diff --git a/packages/core/src/telemetry/config.ts b/packages/core/src/telemetry/config.ts index 335359fafdf..f5548540116 100644 --- a/packages/core/src/telemetry/config.ts +++ b/packages/core/src/telemetry/config.ts @@ -101,8 +101,15 @@ function setHeaderCaseInsensitive( } function validateHeadersObject( - headers: object, + headers: unknown, ): Record | undefined { + if ( + typeof headers !== 'object' || + headers === null || + Array.isArray(headers) + ) { + return undefined; + } const validated: Record = {}; const lowerKeyMap = new Map(); for (const [k, v] of Object.entries(headers)) { @@ -136,18 +143,11 @@ export function parseOtlpHeaders( if (trimmed.startsWith('{')) { try { const parsed: unknown = JSON.parse(trimmed); - if ( - typeof parsed === 'object' && - parsed !== null && - !Array.isArray(parsed) - ) { - const validated = validateHeadersObject(parsed); - if (!validated || Object.keys(validated).length === 0) { - return undefined; - } - return validated; + const validated = validateHeadersObject(parsed); + if (!validated || Object.keys(validated).length === 0) { + return undefined; } - return undefined; + return validated; } catch { return undefined; } @@ -291,7 +291,7 @@ export async function resolveTelemetrySettings(options: { otelEnvHeaders = parseOtlpHeaders(rawOtelEnvHeaders); if (otelEnvHeaders === undefined) { throw new FatalConfigError( - `Invalid telemetry OTLP headers: ${rawOtelEnvHeaders}. Expected JSON object or key=value pairs`, + 'Invalid telemetry OTLP headers in OTEL_EXPORTER_OTLP_HEADERS. Expected JSON object or key=value pairs', ); } } @@ -302,7 +302,7 @@ export async function resolveTelemetrySettings(options: { geminiEnvHeaders = parseOtlpHeaders(rawGeminiEnvHeaders); if (geminiEnvHeaders === undefined) { throw new FatalConfigError( - `Invalid telemetry OTLP headers: ${rawGeminiEnvHeaders}. Expected JSON object or key=value pairs`, + 'Invalid telemetry OTLP headers in GEMINI_TELEMETRY_OTLP_HEADERS. Expected JSON object or key=value pairs', ); } } @@ -314,7 +314,7 @@ export async function resolveTelemetrySettings(options: { argvHeaders = parseOtlpHeaders(argv.telemetryOtlpHeaders); if (argvHeaders === undefined) { throw new FatalConfigError( - `Invalid telemetry OTLP headers: ${argv.telemetryOtlpHeaders}. Expected JSON object or key=value pairs`, + 'Invalid telemetry OTLP headers in argv. Expected JSON object or key=value pairs', ); } } From cdf7f28d507ae38ad6fbfde4c0977bb7cf841db5 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Mon, 5 Oct 2026 19:28:50 +0000 Subject: [PATCH 5/6] fix(telemetry): percent-decode OTLP headers and deduplicate error helpers --- packages/core/src/telemetry/config.test.ts | 13 +++++++ packages/core/src/telemetry/config.ts | 40 +++++++++++++--------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/packages/core/src/telemetry/config.test.ts b/packages/core/src/telemetry/config.test.ts index 43afe6b0af3..524625dfd50 100644 --- a/packages/core/src/telemetry/config.test.ts +++ b/packages/core/src/telemetry/config.test.ts @@ -139,9 +139,22 @@ describe('telemetry/config helpers', () => { it('rejects header values with control characters (CRLF injection)', () => { expect(parseOtlpHeaders('x-key=val\r\nInjected: true')).toBeUndefined(); + expect(parseOtlpHeaders('x-key=val%0D%0AInjected: true')).toBeUndefined(); expect(parseOtlpHeaders('{"x-key": "val\\u0000bad"}')).toBeUndefined(); }); + it('percent-decodes keys and values in key=value format and falls back on malformed encoding', () => { + expect( + parseOtlpHeaders( + 'Authorization=Bearer%20token%2C123,x-custom%2Dkey=a%3Db,x-raw=100%unencoded', + ), + ).toEqual({ + Authorization: 'Bearer token,123', + 'x-custom-key': 'a=b', + 'x-raw': '100%unencoded', + }); + }); + it('skips invalid pairs in key=value format and keeps valid ones', () => { expect( parseOtlpHeaders('Authorization=Bearer token,invalid,=nokey,x-ok=yes'), diff --git a/packages/core/src/telemetry/config.ts b/packages/core/src/telemetry/config.ts index f5548540116..a1eca9ed189 100644 --- a/packages/core/src/telemetry/config.ts +++ b/packages/core/src/telemetry/config.ts @@ -167,8 +167,14 @@ export function parseOtlpHeaders( if (!trimmedPair) continue; const eqIndex = trimmedPair.indexOf('='); if (eqIndex === -1) continue; - const key = trimmedPair.slice(0, eqIndex).trim(); - const val = stripQuotes(trimmedPair.slice(eqIndex + 1).trim()); + let key = trimmedPair.slice(0, eqIndex).trim(); + let val = stripQuotes(trimmedPair.slice(eqIndex + 1).trim()); + try { + key = decodeURIComponent(key); + val = decodeURIComponent(val); + } catch { + // Fallback to undecoded values if decoding fails + } if (key && isValidHeaderName(key) && isValidHeaderValue(val)) { setHeaderCaseInsensitive(headers, lowerKeyMap, key, val); } @@ -216,6 +222,16 @@ export async function resolveTelemetrySettings(options: { ); const settings = options.settings ?? {}; + const createInvalidHeadersError = (source: string) => + new FatalConfigError( + `Invalid telemetry OTLP headers in ${source}. Expected JSON object or key=value pairs`, + ); + + const createInvalidHeaderTokensError = (source: string) => + new FatalConfigError( + `Invalid telemetry OTLP headers in ${source}: header names and values must be valid HTTP header tokens.`, + ); + const enabled = argv.telemetry ?? parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_ENABLED']) ?? @@ -279,9 +295,7 @@ export async function resolveTelemetrySettings(options: { if (settings.otlpHeaders !== undefined) { settingsHeaders = validateHeadersObject(settings.otlpHeaders); if (settingsHeaders === undefined) { - throw new FatalConfigError( - 'Invalid telemetry OTLP headers in settings: header names and values must be valid HTTP header tokens.', - ); + throw createInvalidHeaderTokensError('settings'); } } @@ -290,9 +304,7 @@ export async function resolveTelemetrySettings(options: { if (rawOtelEnvHeaders !== undefined && rawOtelEnvHeaders.trim() !== '') { otelEnvHeaders = parseOtlpHeaders(rawOtelEnvHeaders); if (otelEnvHeaders === undefined) { - throw new FatalConfigError( - 'Invalid telemetry OTLP headers in OTEL_EXPORTER_OTLP_HEADERS. Expected JSON object or key=value pairs', - ); + throw createInvalidHeadersError('OTEL_EXPORTER_OTLP_HEADERS'); } } @@ -301,9 +313,7 @@ export async function resolveTelemetrySettings(options: { if (rawGeminiEnvHeaders !== undefined && rawGeminiEnvHeaders.trim() !== '') { geminiEnvHeaders = parseOtlpHeaders(rawGeminiEnvHeaders); if (geminiEnvHeaders === undefined) { - throw new FatalConfigError( - 'Invalid telemetry OTLP headers in GEMINI_TELEMETRY_OTLP_HEADERS. Expected JSON object or key=value pairs', - ); + throw createInvalidHeadersError('GEMINI_TELEMETRY_OTLP_HEADERS'); } } @@ -313,17 +323,13 @@ export async function resolveTelemetrySettings(options: { if (argv.telemetryOtlpHeaders.trim() !== '') { argvHeaders = parseOtlpHeaders(argv.telemetryOtlpHeaders); if (argvHeaders === undefined) { - throw new FatalConfigError( - 'Invalid telemetry OTLP headers in argv. Expected JSON object or key=value pairs', - ); + throw createInvalidHeadersError('argv'); } } } else { argvHeaders = validateHeadersObject(argv.telemetryOtlpHeaders); if (argvHeaders === undefined) { - throw new FatalConfigError( - 'Invalid telemetry OTLP headers in argv: header names and values must be valid HTTP header tokens.', - ); + throw createInvalidHeaderTokensError('argv'); } } } From 8dd6eaaf8c3f4282c9607d69969d779983af02d8 Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Mon, 5 Oct 2026 20:47:09 +0000 Subject: [PATCH 6/6] fix(telemetry): reject prototype-polluting keys in isValidHeaderName --- packages/core/src/telemetry/config.test.ts | 7 ++++++- packages/core/src/telemetry/config.ts | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/core/src/telemetry/config.test.ts b/packages/core/src/telemetry/config.test.ts index 524625dfd50..6bce6ebdb91 100644 --- a/packages/core/src/telemetry/config.test.ts +++ b/packages/core/src/telemetry/config.test.ts @@ -131,10 +131,15 @@ describe('telemetry/config helpers', () => { expect(parseOtlpHeaders('=value_without_key')).toBeUndefined(); }); - it('rejects header names with invalid RFC 7230 characters', () => { + it('rejects header names with invalid RFC 7230 characters or prototype-polluting keys', () => { expect(parseOtlpHeaders('bad name=value')).toBeUndefined(); expect(parseOtlpHeaders('{"bad name": "value"}')).toBeUndefined(); expect(parseOtlpHeaders('bad:name=value')).toBeUndefined(); + expect(parseOtlpHeaders('__proto__=polluted')).toBeUndefined(); + expect(parseOtlpHeaders('constructor=polluted')).toBeUndefined(); + expect(parseOtlpHeaders('prototype=polluted')).toBeUndefined(); + expect(parseOtlpHeaders('{"__proto__": "polluted"}')).toBeUndefined(); + expect(parseOtlpHeaders('{"constructor": "polluted"}')).toBeUndefined(); }); it('rejects header values with control characters (CRLF injection)', () => { diff --git a/packages/core/src/telemetry/config.ts b/packages/core/src/telemetry/config.ts index a1eca9ed189..8ddab04f322 100644 --- a/packages/core/src/telemetry/config.ts +++ b/packages/core/src/telemetry/config.ts @@ -47,7 +47,13 @@ export interface TelemetryArgOverrides { const HEADER_NAME_REGEX = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; function isValidHeaderName(name: string): boolean { - return HEADER_NAME_REGEX.test(name); + const lower = name.toLowerCase(); + return ( + HEADER_NAME_REGEX.test(name) && + lower !== '__proto__' && + lower !== 'constructor' && + lower !== 'prototype' + ); } function isValidHeaderValue(value: string): boolean {