diff --git a/docs/cli/telemetry.md b/docs/cli/telemetry.md index a1d44eeb235..49198b63167 100644 --- a/docs/cli/telemetry.md +++ b/docs/cli/telemetry.md @@ -42,6 +42,7 @@ Environment variables can override these settings. | `target` | `GEMINI_TELEMETRY_TARGET` | Where to send telemetry data | `"gcp"`/`"local"` | `"local"` | | `otlpEndpoint` | `GEMINI_TELEMETRY_OTLP_ENDPOINT` | OTLP collector endpoint | URL string | `http://localhost:4317` | | `otlpProtocol` | `GEMINI_TELEMETRY_OTLP_PROTOCOL` | OTLP transport protocol | `"grpc"`/`"http"` | `"grpc"` | +| `otlpHeaders` | `GEMINI_TELEMETRY_OTLP_HEADERS` | Custom headers for OTLP exporter requests | JSON/`key=value` | - | | `outfile` | `GEMINI_TELEMETRY_OUTFILE` | Save telemetry to file (overrides `otlpEndpoint`) | file path | - | | `logPrompts` | `GEMINI_TELEMETRY_LOG_PROMPTS` | Include prompts in telemetry logs | `true`/`false` | `true` | | `useCollector` | `GEMINI_TELEMETRY_USE_COLLECTOR` | Use external OTLP collector (advanced) | `true`/`false` | `false` | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 61d158ded90..68db15f69f7 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -2550,6 +2550,8 @@ see [Telemetry](../cli/telemetry.md). - **`otlpEndpoint`** (string): The endpoint for the OTLP Exporter. - **`otlpProtocol`** (string): The protocol for the OTLP Exporter (`grpc` or `http`). + - **`otlpHeaders`** (object): Custom headers to send with OTLP Exporter + requests. - **`logPrompts`** (boolean): Whether or not to include the content of user prompts in the logs. - **`outfile`** (string): The file to write telemetry to when `target` is @@ -2766,6 +2768,11 @@ the `advanced.excludedEnvVars` setting in your `settings.json` file. - **`GEMINI_TELEMETRY_OTLP_PROTOCOL`**: - Sets the OTLP protocol (`grpc` or `http`). - Overrides the `telemetry.otlpProtocol` setting. +- **`GEMINI_TELEMETRY_OTLP_HEADERS`**: + - Sets custom headers for OTLP exporter requests (as a JSON object string or + comma-separated `key=value` pairs; also falls back to + `OTEL_EXPORTER_OTLP_HEADERS`). + - Merges with and overrides headers in the `telemetry.otlpHeaders` setting. - **`GEMINI_TELEMETRY_LOG_PROMPTS`**: - Set to `true` or `1` to enable or disable logging of user prompts. Any other value is treated as disabling it. diff --git a/packages/cli/src/config/settings.test.ts b/packages/cli/src/config/settings.test.ts index e445752b085..53e81d00c5c 100644 --- a/packages/cli/src/config/settings.test.ts +++ b/packages/cli/src/config/settings.test.ts @@ -903,6 +903,34 @@ describe('Settings Loading and Merging', () => { expect(settings.merged.telemetry?.enabled).toBe(false); }); + it('should load telemetry.otlpHeaders from settings', () => { + (mockFsExistsSync as Mock).mockImplementation( + (p: fs.PathLike) => + normalizePath(p) === normalizePath(USER_SETTINGS_PATH), + ); + const userSettingsContent = { + telemetry: { + enabled: true, + otlpHeaders: { + Authorization: 'Bearer test-token-value', + 'x-custom': 'static-val', + }, + }, + }; + (fs.readFileSync as Mock).mockImplementation( + (p: fs.PathOrFileDescriptor) => { + if (normalizePath(p) === normalizePath(USER_SETTINGS_PATH)) + return JSON.stringify(userSettingsContent); + return '{}'; + }, + ); + const settings = loadSettings(MOCK_WORKSPACE_DIR); + expect(settings.merged.telemetry?.otlpHeaders).toEqual({ + Authorization: 'Bearer test-token-value', + 'x-custom': 'static-val', + }); + }); + it('should have telemetry as undefined if not in any settings file', () => { (mockFsExistsSync as Mock).mockReturnValue(false); // No settings files exist (fs.readFileSync as Mock).mockReturnValue('{}'); diff --git a/packages/cli/src/config/settingsSchema.ts b/packages/cli/src/config/settingsSchema.ts index 44a830bc545..f85279cdc3d 100644 --- a/packages/cli/src/config/settingsSchema.ts +++ b/packages/cli/src/config/settingsSchema.ts @@ -3239,6 +3239,11 @@ export const SETTINGS_SCHEMA_DEFINITIONS: Record< description: 'Protocol for OTLP exporters.', enum: ['grpc', 'http'], }, + otlpHeaders: { + type: 'object', + description: 'Custom headers to send with OTLP exporter requests.', + additionalProperties: { type: 'string' }, + }, traces: { type: 'boolean', description: diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index e84c41f7d11..6775b67799f 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -1385,6 +1385,32 @@ describe('Server Config (config.ts)', () => { const config = new Config(paramsWithoutTelemetry); expect(config.getTelemetryOtlpProtocol()).toBe('grpc'); }); + + it('should return provided OTLP headers', () => { + const headers = { Authorization: 'Bearer token123' }; + const params: ConfigParameters = { + ...baseParams, + telemetry: { enabled: true, otlpHeaders: headers }, + }; + const config = new Config(params); + expect(config.getTelemetryOtlpHeaders()).toEqual(headers); + }); + + it('should return empty object for OTLP headers if not provided', () => { + const params: ConfigParameters = { + ...baseParams, + telemetry: { enabled: true }, + }; + const config = new Config(params); + expect(config.getTelemetryOtlpHeaders()).toEqual({}); + }); + + it('should return empty object for OTLP headers if telemetry object is not provided', () => { + const paramsWithoutTelemetry: ConfigParameters = { ...baseParams }; + delete paramsWithoutTelemetry.telemetry; + const config = new Config(paramsWithoutTelemetry); + expect(config.getTelemetryOtlpHeaders()).toEqual({}); + }); }); describe('UseRipgrep Configuration', () => { diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 6b8649e3500..5cb9d1c8220 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -222,6 +222,7 @@ export interface TelemetrySettings { target?: TelemetryTarget; otlpEndpoint?: string; otlpProtocol?: 'grpc' | 'http'; + otlpHeaders?: Record; logPrompts?: boolean; outfile?: string; useCollector?: boolean; @@ -1096,6 +1097,7 @@ export class Config implements McpContext, AgentLoopContext { target: params.telemetry?.target ?? DEFAULT_TELEMETRY_TARGET, otlpEndpoint: params.telemetry?.otlpEndpoint ?? DEFAULT_OTLP_ENDPOINT, otlpProtocol: params.telemetry?.otlpProtocol, + otlpHeaders: params.telemetry?.otlpHeaders, logPrompts: params.telemetry?.logPrompts ?? true, outfile: params.telemetry?.outfile, useCollector: params.telemetry?.useCollector, @@ -2922,6 +2924,10 @@ export class Config implements McpContext, AgentLoopContext { return this.telemetrySettings.otlpProtocol ?? 'grpc'; } + getTelemetryOtlpHeaders(): Record { + return this.telemetrySettings.otlpHeaders ?? {}; + } + getTelemetryTarget(): TelemetryTarget { return this.telemetrySettings.target ?? DEFAULT_TELEMETRY_TARGET; } diff --git a/packages/core/src/telemetry/config.test.ts b/packages/core/src/telemetry/config.test.ts index 232aa274667..6bce6ebdb91 100644 --- a/packages/core/src/telemetry/config.test.ts +++ b/packages/core/src/telemetry/config.test.ts @@ -7,6 +7,7 @@ import { describe, it, expect } from 'vitest'; import { parseBooleanEnvFlag, + parseOtlpHeaders, parseTelemetryTargetValue, resolveTelemetrySettings, } from './config.js'; @@ -53,6 +54,122 @@ describe('telemetry/config helpers', () => { }); }); + describe('parseOtlpHeaders', () => { + it('returns undefined for undefined or empty string', () => { + expect(parseOtlpHeaders(undefined)).toBeUndefined(); + expect(parseOtlpHeaders('')).toBeUndefined(); + expect(parseOtlpHeaders(' ')).toBeUndefined(); + }); + + it('parses JSON object format', () => { + const json = '{"Authorization":"Bearer token123","x-api-key":"abc"}'; + expect(parseOtlpHeaders(json)).toEqual({ + Authorization: 'Bearer token123', + 'x-api-key': 'abc', + }); + }); + + it('returns undefined for empty JSON object', () => { + expect(parseOtlpHeaders('{}')).toBeUndefined(); + }); + + it('returns undefined for malformed JSON or non-string values without falling back to key=value', () => { + expect(parseOtlpHeaders('{Authorization=Bearer token}')).toBeUndefined(); + expect(parseOtlpHeaders('{"key": 123}')).toBeUndefined(); + expect(parseOtlpHeaders('{"key": null}')).toBeUndefined(); + expect(parseOtlpHeaders('{"key": true}')).toBeUndefined(); + expect(parseOtlpHeaders('{"valid": "yes", "bad": 42}')).toBeUndefined(); + expect(parseOtlpHeaders('["a", "b"]')).toBeUndefined(); + }); + + it('parses comma-separated key=value pairs', () => { + expect( + parseOtlpHeaders('Authorization=Bearer token123,x-api-key=abc'), + ).toEqual({ + Authorization: 'Bearer token123', + 'x-api-key': 'abc', + }); + }); + + it('preserves semicolons inside header values in key=value format', () => { + expect( + parseOtlpHeaders( + 'Authorization=Bearer token123;tenant=abc,x-api-key=xyz', + ), + ).toEqual({ + Authorization: 'Bearer token123;tenant=abc', + 'x-api-key': 'xyz', + }); + }); + + it('handles values containing equals signs (e.g. base64)', () => { + expect(parseOtlpHeaders('Authorization=Basic dXNlcjpwYXNz==')).toEqual({ + Authorization: 'Basic dXNlcjpwYXNz==', + }); + }); + + it('strips surrounding quotes from values in key=value format', () => { + expect( + parseOtlpHeaders('Authorization="Bearer token",x-key=\'val\''), + ).toEqual({ + Authorization: 'Bearer token', + 'x-key': 'val', + }); + }); + + it('trims whitespace around keys and values', () => { + expect( + parseOtlpHeaders(' Authorization = Bearer token , x-key = val '), + ).toEqual({ + Authorization: 'Bearer token', + 'x-key': 'val', + }); + }); + + it('returns undefined for invalid key=value strings', () => { + expect(parseOtlpHeaders('invalid-no-equals')).toBeUndefined(); + expect(parseOtlpHeaders('=value_without_key')).toBeUndefined(); + }); + + it('rejects header names with invalid RFC 7230 characters or prototype-polluting keys', () => { + expect(parseOtlpHeaders('bad name=value')).toBeUndefined(); + expect(parseOtlpHeaders('{"bad name": "value"}')).toBeUndefined(); + expect(parseOtlpHeaders('bad:name=value')).toBeUndefined(); + expect(parseOtlpHeaders('__proto__=polluted')).toBeUndefined(); + expect(parseOtlpHeaders('constructor=polluted')).toBeUndefined(); + expect(parseOtlpHeaders('prototype=polluted')).toBeUndefined(); + expect(parseOtlpHeaders('{"__proto__": "polluted"}')).toBeUndefined(); + expect(parseOtlpHeaders('{"constructor": "polluted"}')).toBeUndefined(); + }); + + it('rejects header values with control characters (CRLF injection)', () => { + expect(parseOtlpHeaders('x-key=val\r\nInjected: true')).toBeUndefined(); + expect(parseOtlpHeaders('x-key=val%0D%0AInjected: true')).toBeUndefined(); + expect(parseOtlpHeaders('{"x-key": "val\\u0000bad"}')).toBeUndefined(); + }); + + it('percent-decodes keys and values in key=value format and falls back on malformed encoding', () => { + expect( + parseOtlpHeaders( + 'Authorization=Bearer%20token%2C123,x-custom%2Dkey=a%3Db,x-raw=100%unencoded', + ), + ).toEqual({ + Authorization: 'Bearer token,123', + 'x-custom-key': 'a=b', + 'x-raw': '100%unencoded', + }); + }); + + it('skips invalid pairs in key=value format and keeps valid ones', () => { + expect( + parseOtlpHeaders('Authorization=Bearer token,invalid,=nokey,x-ok=yes'), + ).toEqual({ + Authorization: 'Bearer token', + 'x-ok': 'yes', + }); + }); + }); + describe('resolveTelemetrySettings', () => { it('falls back to settings when no argv/env provided', async () => { const settings = { @@ -179,5 +296,117 @@ describe('telemetry/config helpers', () => { /Invalid telemetry target/i, ); }); + + it('resolves otlpHeaders from settings', async () => { + const settings = { + otlpHeaders: { Authorization: 'Bearer settings-token' }, + }; + const resolved = await resolveTelemetrySettings({ settings }); + expect(resolved.otlpHeaders).toEqual({ + Authorization: 'Bearer settings-token', + }); + }); + + it('resolves otlpHeaders from GEMINI_TELEMETRY_OTLP_HEADERS and OTEL_EXPORTER_OTLP_HEADERS', async () => { + const resolvedOtel = await resolveTelemetrySettings({ + env: { OTEL_EXPORTER_OTLP_HEADERS: 'x-otel=1,Authorization=Bearer a' }, + }); + expect(resolvedOtel.otlpHeaders).toEqual({ + 'x-otel': '1', + Authorization: 'Bearer a', + }); + + const resolvedGemini = await resolveTelemetrySettings({ + env: { + OTEL_EXPORTER_OTLP_HEADERS: 'x-otel=1,Authorization=Bearer a', + GEMINI_TELEMETRY_OTLP_HEADERS: '{"Authorization":"Bearer b"}', + }, + }); + expect(resolvedGemini.otlpHeaders).toEqual({ + 'x-otel': '1', + Authorization: 'Bearer b', + }); + }); + + it('merges otlpHeaders across settings, env, and argv with case-insensitive deduplication', async () => { + const settings = { + otlpHeaders: { + Authorization: 'Bearer settings', + 'X-Settings': 'from-settings', + }, + }; + const env = { + GEMINI_TELEMETRY_OTLP_HEADERS: + 'authorization=Bearer env,X-Env=from-env', + }; + const argv = { + telemetryOtlpHeaders: 'X-Argv=from-argv', + }; + const resolved = await resolveTelemetrySettings({ argv, env, settings }); + expect(resolved.otlpHeaders).toEqual({ + 'X-Settings': 'from-settings', + authorization: 'Bearer env', + 'X-Env': 'from-env', + 'X-Argv': 'from-argv', + }); + }); + + it('throws FatalConfigError on invalid otlpHeaders without leaking raw secret values', async () => { + await expect( + resolveTelemetrySettings({ + env: { + GEMINI_TELEMETRY_OTLP_HEADERS: '{"Authorization":"secret-token-1"', + }, + }), + ).rejects.toThrow( + 'Invalid telemetry OTLP headers in GEMINI_TELEMETRY_OTLP_HEADERS. Expected JSON object or key=value pairs', + ); + + await expect( + resolveTelemetrySettings({ + env: { + OTEL_EXPORTER_OTLP_HEADERS: '{"Authorization":"secret-token-2"', + }, + }), + ).rejects.toThrow( + 'Invalid telemetry OTLP headers in OTEL_EXPORTER_OTLP_HEADERS. Expected JSON object or key=value pairs', + ); + + await expect( + resolveTelemetrySettings({ + argv: { telemetryOtlpHeaders: '{"Authorization":"secret-token-3"' }, + }), + ).rejects.toThrow( + 'Invalid telemetry OTLP headers in argv. Expected JSON object or key=value pairs', + ); + + await expect( + resolveTelemetrySettings({ + settings: { otlpHeaders: { 'bad name': 'value' } }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in settings/i); + + await expect( + resolveTelemetrySettings({ + settings: { + otlpHeaders: null as unknown as Record, + }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in settings/i); + + await expect( + resolveTelemetrySettings({ + argv: { telemetryOtlpHeaders: { 'x-key': 'bad\r\nvalue' } }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in argv/i); + + await expect( + resolveTelemetrySettings({ + argv: { + telemetryOtlpHeaders: ['a=b'] as unknown as Record, + }, + }), + ).rejects.toThrow(/Invalid telemetry OTLP headers in argv/i); + }); }); }); diff --git a/packages/core/src/telemetry/config.ts b/packages/core/src/telemetry/config.ts index 9fd4bacfc3b..8ddab04f322 100644 --- a/packages/core/src/telemetry/config.ts +++ b/packages/core/src/telemetry/config.ts @@ -39,10 +39,179 @@ export interface TelemetryArgOverrides { telemetryTarget?: string | TelemetryTarget; telemetryOtlpEndpoint?: string; telemetryOtlpProtocol?: string; + telemetryOtlpHeaders?: string | Record; telemetryLogPrompts?: boolean; telemetryOutfile?: string; } +const HEADER_NAME_REGEX = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; + +function isValidHeaderName(name: string): boolean { + const lower = name.toLowerCase(); + return ( + HEADER_NAME_REGEX.test(name) && + lower !== '__proto__' && + lower !== 'constructor' && + lower !== 'prototype' + ); +} + +function isValidHeaderValue(value: string): boolean { + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if ((code < 0x20 && code !== 0x09) || code === 0x7f) { + return false; + } + } + return true; +} + +function stripQuotes(value: string): string { + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + return value.slice(1, -1); + } + return value; +} + +const ALLOWED_TELEMETRY_ENV_KEYS = new Set([ + 'GEMINI_TELEMETRY_ENABLED', + 'GEMINI_TELEMETRY_TRACES_ENABLED', + 'GEMINI_TELEMETRY_TARGET', + 'GEMINI_TELEMETRY_OTLP_ENDPOINT', + 'OTEL_EXPORTER_OTLP_ENDPOINT', + 'GEMINI_TELEMETRY_OTLP_PROTOCOL', + 'GEMINI_TELEMETRY_OTLP_HEADERS', + 'OTEL_EXPORTER_OTLP_HEADERS', + 'GEMINI_TELEMETRY_LOG_PROMPTS', + 'GEMINI_TELEMETRY_OUTFILE', + 'GEMINI_TELEMETRY_USE_COLLECTOR', + 'GEMINI_TELEMETRY_USE_CLI_AUTH', +]); + +function setHeaderCaseInsensitive( + headers: Record, + lowerKeyMap: Map, + key: string, + value: string, +): void { + const lowerKey = key.toLowerCase(); + const existingKey = lowerKeyMap.get(lowerKey); + if (existingKey !== undefined && existingKey !== key) { + delete headers[existingKey]; + } + headers[key] = value; + lowerKeyMap.set(lowerKey, key); +} + +function validateHeadersObject( + headers: unknown, +): Record | undefined { + if ( + typeof headers !== 'object' || + headers === null || + Array.isArray(headers) + ) { + return undefined; + } + const validated: Record = {}; + const lowerKeyMap = new Map(); + for (const [k, v] of Object.entries(headers)) { + const trimmedKey = k.trim(); + if ( + typeof v !== 'string' || + !trimmedKey || + !isValidHeaderName(trimmedKey) || + !isValidHeaderValue(v) + ) { + return undefined; + } + setHeaderCaseInsensitive(validated, lowerKeyMap, trimmedKey, v); + } + return validated; +} + +/** + * Parse OTLP headers from a string. + * Supports JSON object format (e.g., '{"Authorization":"Bearer token"}') or + * key=value pairs separated by commas (e.g., 'Authorization=Bearer token,x-api-key=abc123'). + */ +export function parseOtlpHeaders( + value: string | undefined, +): Record | undefined { + if (!value || value.trim() === '') return undefined; + + const trimmed = value.trim(); + + // If the string starts with '{', treat it strictly as JSON and do not fall through to key=value parsing. + if (trimmed.startsWith('{')) { + try { + const parsed: unknown = JSON.parse(trimmed); + const validated = validateHeadersObject(parsed); + if (!validated || Object.keys(validated).length === 0) { + return undefined; + } + return validated; + } catch { + return undefined; + } + } + + // Reject other JSON literals like arrays + if (trimmed.startsWith('[')) { + return undefined; + } + + // Parse as key=value pairs separated by commas + const headers: Record = {}; + const lowerKeyMap = new Map(); + const pairs = trimmed.split(','); + for (const pair of pairs) { + const trimmedPair = pair.trim(); + if (!trimmedPair) continue; + const eqIndex = trimmedPair.indexOf('='); + if (eqIndex === -1) continue; + let key = trimmedPair.slice(0, eqIndex).trim(); + let val = stripQuotes(trimmedPair.slice(eqIndex + 1).trim()); + try { + key = decodeURIComponent(key); + val = decodeURIComponent(val); + } catch { + // Fallback to undecoded values if decoding fails + } + if (key && isValidHeaderName(key) && isValidHeaderValue(val)) { + setHeaderCaseInsensitive(headers, lowerKeyMap, key, val); + } + } + + return Object.keys(headers).length > 0 ? headers : undefined; +} + +/** + * Merge header objects from lowest to highest precedence, deduplicating keys case-insensitively. + */ +function mergeHeaders( + ...sources: Array | undefined> +): Record | undefined { + const definedSources = sources.filter( + (s): s is Record => s !== undefined, + ); + if (definedSources.length === 0) { + return undefined; + } + + const merged: Record = {}; + const lowerKeyMap = new Map(); + for (const source of definedSources) { + for (const [key, value] of Object.entries(source)) { + setHeaderCaseInsensitive(merged, lowerKeyMap, key, value); + } + } + return merged; +} + /** * Build TelemetrySettings by resolving from argv (highest), env, then settings. */ @@ -52,21 +221,35 @@ export async function resolveTelemetrySettings(options: { settings?: TelemetrySettings; }): Promise { const argv = options.argv ?? {}; - const env = options.env ?? {}; + const sanitizedEnv = Object.fromEntries( + Object.entries(options.env ?? {}).filter(([key]) => + ALLOWED_TELEMETRY_ENV_KEYS.has(key), + ), + ); const settings = options.settings ?? {}; + const createInvalidHeadersError = (source: string) => + new FatalConfigError( + `Invalid telemetry OTLP headers in ${source}. Expected JSON object or key=value pairs`, + ); + + const createInvalidHeaderTokensError = (source: string) => + new FatalConfigError( + `Invalid telemetry OTLP headers in ${source}: header names and values must be valid HTTP header tokens.`, + ); + const enabled = argv.telemetry ?? - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_ENABLED']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_ENABLED']) ?? settings.enabled; const traces = - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_TRACES_ENABLED']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_TRACES_ENABLED']) ?? settings.traces; const rawTarget = argv.telemetryTarget ?? - env['GEMINI_TELEMETRY_TARGET'] ?? + sanitizedEnv['GEMINI_TELEMETRY_TARGET'] ?? (settings.target as string | TelemetryTarget | undefined); const target = parseTelemetryTargetValue(rawTarget); if (rawTarget !== undefined && target === undefined) { @@ -79,13 +262,13 @@ export async function resolveTelemetrySettings(options: { const otlpEndpoint = argv.telemetryOtlpEndpoint ?? - env['GEMINI_TELEMETRY_OTLP_ENDPOINT'] ?? - env['OTEL_EXPORTER_OTLP_ENDPOINT'] ?? + sanitizedEnv['GEMINI_TELEMETRY_OTLP_ENDPOINT'] ?? + sanitizedEnv['OTEL_EXPORTER_OTLP_ENDPOINT'] ?? settings.otlpEndpoint; const rawProtocol = argv.telemetryOtlpProtocol ?? - env['GEMINI_TELEMETRY_OTLP_PROTOCOL'] ?? + sanitizedEnv['GEMINI_TELEMETRY_OTLP_PROTOCOL'] ?? settings.otlpProtocol; const otlpProtocol = (['grpc', 'http'] as const).find( (p) => p === rawProtocol, @@ -100,29 +283,82 @@ export async function resolveTelemetrySettings(options: { const logPrompts = argv.telemetryLogPrompts ?? - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_LOG_PROMPTS']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_LOG_PROMPTS']) ?? settings.logPrompts; const outfile = argv.telemetryOutfile ?? - env['GEMINI_TELEMETRY_OUTFILE'] ?? + sanitizedEnv['GEMINI_TELEMETRY_OUTFILE'] ?? settings.outfile; const useCollector = - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_USE_COLLECTOR']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_USE_COLLECTOR']) ?? settings.useCollector; + // Resolve OTLP headers: merge settings (lowest), OTEL_EXPORTER_OTLP_HEADERS, + // GEMINI_TELEMETRY_OTLP_HEADERS, and argv (highest). + let settingsHeaders: Record | undefined; + if (settings.otlpHeaders !== undefined) { + settingsHeaders = validateHeadersObject(settings.otlpHeaders); + if (settingsHeaders === undefined) { + throw createInvalidHeaderTokensError('settings'); + } + } + + const rawOtelEnvHeaders = sanitizedEnv['OTEL_EXPORTER_OTLP_HEADERS']; + let otelEnvHeaders: Record | undefined; + if (rawOtelEnvHeaders !== undefined && rawOtelEnvHeaders.trim() !== '') { + otelEnvHeaders = parseOtlpHeaders(rawOtelEnvHeaders); + if (otelEnvHeaders === undefined) { + throw createInvalidHeadersError('OTEL_EXPORTER_OTLP_HEADERS'); + } + } + + const rawGeminiEnvHeaders = sanitizedEnv['GEMINI_TELEMETRY_OTLP_HEADERS']; + let geminiEnvHeaders: Record | undefined; + if (rawGeminiEnvHeaders !== undefined && rawGeminiEnvHeaders.trim() !== '') { + geminiEnvHeaders = parseOtlpHeaders(rawGeminiEnvHeaders); + if (geminiEnvHeaders === undefined) { + throw createInvalidHeadersError('GEMINI_TELEMETRY_OTLP_HEADERS'); + } + } + + let argvHeaders: Record | undefined; + if (argv.telemetryOtlpHeaders !== undefined) { + if (typeof argv.telemetryOtlpHeaders === 'string') { + if (argv.telemetryOtlpHeaders.trim() !== '') { + argvHeaders = parseOtlpHeaders(argv.telemetryOtlpHeaders); + if (argvHeaders === undefined) { + throw createInvalidHeadersError('argv'); + } + } + } else { + argvHeaders = validateHeadersObject(argv.telemetryOtlpHeaders); + if (argvHeaders === undefined) { + throw createInvalidHeaderTokensError('argv'); + } + } + } + + const otlpHeaders = mergeHeaders( + settingsHeaders, + otelEnvHeaders, + geminiEnvHeaders, + argvHeaders, + ); + return { enabled, traces, target, otlpEndpoint, otlpProtocol, + otlpHeaders, logPrompts, outfile, useCollector, useCliAuth: - parseBooleanEnvFlag(env['GEMINI_TELEMETRY_USE_CLI_AUTH']) ?? + parseBooleanEnvFlag(sanitizedEnv['GEMINI_TELEMETRY_USE_CLI_AUTH']) ?? settings.useCliAuth, }; } diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts index 83e5517882f..d1ca14d276b 100644 --- a/packages/core/src/telemetry/index.ts +++ b/packages/core/src/telemetry/index.ts @@ -23,6 +23,7 @@ export { resolveTelemetrySettings, parseBooleanEnvFlag, parseTelemetryTargetValue, + parseOtlpHeaders, } from './config.js'; export { GcpTraceExporter, diff --git a/packages/core/src/telemetry/sdk.test.ts b/packages/core/src/telemetry/sdk.test.ts index 9636212e2c9..ab250416706 100644 --- a/packages/core/src/telemetry/sdk.test.ts +++ b/packages/core/src/telemetry/sdk.test.ts @@ -17,6 +17,7 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; import { OTLPTraceExporter as OTLPTraceExporterHttp } from '@opentelemetry/exporter-trace-otlp-http'; import { OTLPLogExporter as OTLPLogExporterHttp } from '@opentelemetry/exporter-logs-otlp-http'; import { OTLPMetricExporter as OTLPMetricExporterHttp } from '@opentelemetry/exporter-metrics-otlp-http'; +import { Metadata } from '@grpc/grpc-js'; import { ConsoleSpanExporter } from '@opentelemetry/sdk-trace-node'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { GoogleAuth, type JWTInput } from 'google-auth-library'; @@ -68,6 +69,7 @@ describe('Telemetry SDK', () => { getTelemetryEnabled: () => true, getTelemetryOtlpEndpoint: () => 'http://localhost:4317', getTelemetryOtlpProtocol: () => 'grpc', + getTelemetryOtlpHeaders: () => ({}), getTelemetryTarget: () => 'local', getTelemetryUseCollector: () => false, getTelemetryOutfile: () => undefined, @@ -124,6 +126,99 @@ describe('Telemetry SDK', () => { expect(NodeSDK.prototype.start).toHaveBeenCalled(); }); + it('should pass custom headers to HTTP exporters when otlpHeaders is configured', async () => { + const customHeaders = { + Authorization: 'Bearer secret-token', + 'x-api-key': 'my-api-key', + }; + vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('http'); + vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( + 'http://localhost:4318', + ); + vi.spyOn(mockConfig, 'getTelemetryOtlpHeaders').mockReturnValue( + customHeaders, + ); + + await initializeTelemetry(mockConfig); + + expect(OTLPTraceExporterHttp).toHaveBeenCalledWith({ + url: 'http://localhost:4318/v1/traces', + headers: customHeaders, + }); + expect(OTLPLogExporterHttp).toHaveBeenCalledWith({ + url: 'http://localhost:4318/v1/logs', + headers: customHeaders, + }); + expect(OTLPMetricExporterHttp).toHaveBeenCalledWith({ + url: 'http://localhost:4318/v1/metrics', + headers: customHeaders, + }); + }); + + it('should pass custom metadata to gRPC exporters when otlpHeaders is configured', async () => { + const customHeaders = { + Authorization: 'Bearer secret-token', + 'x-api-key': 'my-api-key', + }; + vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('grpc'); + vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( + 'http://localhost:4317', + ); + vi.spyOn(mockConfig, 'getTelemetryOtlpHeaders').mockReturnValue( + customHeaders, + ); + + await initializeTelemetry(mockConfig); + + for (const ExporterMock of [ + OTLPTraceExporter, + OTLPLogExporter, + OTLPMetricExporter, + ]) { + expect(ExporterMock).toHaveBeenCalledTimes(1); + const callArg = vi.mocked(ExporterMock).mock.calls[0]?.[0] as { + url?: string; + compression?: string; + metadata?: Metadata; + }; + expect(callArg?.url).toBe('http://localhost:4317'); + expect(callArg?.compression).toBe('gzip'); + expect(callArg?.metadata).toBeInstanceOf(Metadata); + expect(callArg?.metadata?.get('authorization')).toEqual([ + 'Bearer secret-token', + ]); + expect(callArg?.metadata?.get('x-api-key')).toEqual(['my-api-key']); + } + }); + + it('should skip invalid gRPC metadata keys and log a warning without crashing', async () => { + const customHeaders = { + bad$key: 'invalid-for-grpc', + 'x-valid-key': 'valid-value', + }; + vi.spyOn(mockConfig, 'getTelemetryOtlpProtocol').mockReturnValue('grpc'); + vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( + 'http://localhost:4317', + ); + vi.spyOn(mockConfig, 'getTelemetryOtlpHeaders').mockReturnValue( + customHeaders, + ); + + await initializeTelemetry(mockConfig); + + expect(debugLogger.warn).toHaveBeenCalledWith( + expect.stringContaining( + '[Telemetry] Skipping invalid gRPC metadata key "bad$key":', + ), + expect.any(Error), + ); + const callArg = vi.mocked(OTLPTraceExporter).mock.calls[0]?.[0] as { + metadata?: Metadata; + }; + expect(callArg?.metadata).toBeInstanceOf(Metadata); + expect(callArg?.metadata?.get('x-valid-key')).toEqual(['valid-value']); + }); + it('should parse gRPC endpoint correctly', async () => { vi.spyOn(mockConfig, 'getTelemetryOtlpEndpoint').mockReturnValue( 'https://my-collector.com', diff --git a/packages/core/src/telemetry/sdk.ts b/packages/core/src/telemetry/sdk.ts index ac90bf86ad8..d3f3495f563 100644 --- a/packages/core/src/telemetry/sdk.ts +++ b/packages/core/src/telemetry/sdk.ts @@ -19,6 +19,7 @@ import { OTLPTraceExporter as OTLPTraceExporterHttp } from '@opentelemetry/expor import { OTLPLogExporter as OTLPLogExporterHttp } from '@opentelemetry/exporter-logs-otlp-http'; import { OTLPMetricExporter as OTLPMetricExporterHttp } from '@opentelemetry/exporter-metrics-otlp-http'; import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base'; +import { Metadata } from '@grpc/grpc-js'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import { resourceFromAttributes } from '@opentelemetry/resources'; @@ -279,6 +280,9 @@ export async function initializeTelemetry( exportIntervalMillis: 30000, }); } else if (useOtlp) { + const otlpHeaders = config.getTelemetryOtlpHeaders?.() ?? {}; + const hasHeaders = Object.keys(otlpHeaders).length > 0; + if (otlpProtocol === 'http') { const buildUrl = (path: string) => { const url = new URL(parsedEndpoint); @@ -288,30 +292,55 @@ export async function initializeTelemetry( }; spanExporter = new OTLPTraceExporterHttp({ url: buildUrl('v1/traces'), + headers: hasHeaders ? otlpHeaders : undefined, }); logExporter = new OTLPLogExporterHttp({ url: buildUrl('v1/logs'), + headers: hasHeaders ? otlpHeaders : undefined, }); metricReader = new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporterHttp({ url: buildUrl('v1/metrics'), + headers: hasHeaders ? otlpHeaders : undefined, }), exportIntervalMillis: 10000, }); } else { // grpc + let metadata: Metadata | undefined; + if (hasHeaders) { + const candidateMetadata = new Metadata(); + let validCount = 0; + for (const [key, value] of Object.entries(otlpHeaders)) { + try { + candidateMetadata.set(key, value); + validCount++; + } catch (error) { + debugLogger.warn( + `[Telemetry] Skipping invalid gRPC metadata key "${key}":`, + error, + ); + } + } + if (validCount > 0) { + metadata = candidateMetadata; + } + } spanExporter = new OTLPTraceExporter({ url: parsedEndpoint, compression: CompressionAlgorithm.GZIP, + ...(metadata && { metadata }), }); logExporter = new OTLPLogExporter({ url: parsedEndpoint, compression: CompressionAlgorithm.GZIP, + ...(metadata && { metadata }), }); metricReader = new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter({ url: parsedEndpoint, compression: CompressionAlgorithm.GZIP, + ...(metadata && { metadata }), }), exportIntervalMillis: 10000, }); diff --git a/schemas/settings.schema.json b/schemas/settings.schema.json index 6c95ee8b492..fe0f11c45be 100644 --- a/schemas/settings.schema.json +++ b/schemas/settings.schema.json @@ -4282,6 +4282,13 @@ "description": "Protocol for OTLP exporters.", "enum": ["grpc", "http"] }, + "otlpHeaders": { + "type": "object", + "description": "Custom headers to send with OTLP exporter requests.", + "additionalProperties": { + "type": "string" + } + }, "traces": { "type": "boolean", "description": "Whether detailed traces with large attributes are captured."