From c092e6d983850dae0b6a5f96b9ce10876d25325f Mon Sep 17 00:00:00 2001 From: Starfolk Date: Tue, 21 Jul 2026 01:08:42 +0000 Subject: [PATCH 1/3] fix(openai): allowlist metadata pass-through, drop eager response serialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Aligns the OpenAI integration with `.agents/skills/sdk-integrations/SKILL.md`, following the same pattern as #604 (mistral), #594 (litellm), #603 (livekit), and #583 (anthropic). - **Allowlist instead of denylist for metadata.** Every `_parse_params` copy (`ChatCompletionWrapper`, `ResponseWrapper`, `BaseWrapper` for Embedding/ Moderation/Speech, `_AudioFileWrapper`, `_ImageBaseWrapper`) previously did `metadata: {**params, "provider": "openai"}` — every non-input request kwarg landed in span metadata. `ResponseWrapper._parse_event_from_result` did the same on the response side. That leaked `extra_headers`, `extra_query`, `extra_body`, user-provided Responses `metadata`, `user`, `safety_identifier`, `prompt_cache_key`, and anything OpenAI adds in future SDK versions straight into telemetry. Replaced with per-surface allowlist tuples plus a shared `_filter_metadata` helper that drops `NOT_GIVEN`/`Omit` sentinels and serializes `response_format` inline. Mirrors Anthropic's `METADATA_PARAMS`. - **Drop eager response serialization.** `_try_to_dict(raw_response)` was full-dumping every non-streaming Chat/Response API response just to read `usage`/`choices`/`output` — `bt_safe_deep_copy` already dumps at log time. Now reads via `getattr`; a narrow `_choices_output` helper dumps only `choices` (needed for the audio-attachment walker). `_parse_event_from_result` and `_extract_moderation_metadata` were made dict-or-object aware. - **Note on `moderation`.** Added to both chat + responses request allowlists and the responses result allowlist because it's a Braintrust-proxy request parameter that deep-merges with the response's moderation object at log time. Three tests that asserted `extra_headers` leaked into metadata (`extra_headers ["X-Stainless-Helper-Method"] == "chat.completions.stream"`) flipped to assert the leak is now blocked. Test plan: - No new mocks/fakes; no cassette re-records. - `nox -s "test_openai(latest)"` — 76/76 pass - `nox -s "test_openai(1.92.0)"`, `test_openai(1.77.0)`, `test_openai(1.71.0)` — all green - `nox -s "test_openai_http2_streaming(latest)"` — 3/3 - `nox -s "test_btx_openai(latest)"` — 5/5 - `nox -s "test_openai_agents(latest)"` — 7/7 - `ruff check` + `ruff format --check` clean Co-Authored-By: Claude Opus 4.7 --- .../integrations/openai/test_openai.py | 6 +- .../braintrust/integrations/openai/tracing.py | 256 +++++++++++++----- 2 files changed, 196 insertions(+), 66 deletions(-) diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index b95ce084..578fd0c9 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -632,7 +632,7 @@ def test_openai_chat_stream_helper_sync(memory_logger): metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert span["metadata"]["stream"] == True - assert span["metadata"]["extra_headers"]["X-Stainless-Helper-Method"] == "chat.completions.stream" + assert "extra_headers" not in span["metadata"] assert TEST_MODEL in span["metadata"]["model"] assert span["metadata"]["provider"] == "openai" assert TEST_PROMPT in str(span["input"]) @@ -1180,7 +1180,7 @@ async def test_openai_chat_stream_helper_async(memory_logger): metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert span["metadata"]["stream"] == True - assert span["metadata"]["extra_headers"]["X-Stainless-Helper-Method"] == "chat.completions.stream" + assert "extra_headers" not in span["metadata"] assert TEST_MODEL in span["metadata"]["model"] assert span["metadata"]["provider"] == "openai" assert TEST_PROMPT in str(span["input"]) @@ -2751,7 +2751,7 @@ def test_setup_stream_helper_creates_spans(self, memory_logger): span = spans[0] assert_metrics_are_valid(span["metrics"], start, end) assert span["metadata"]["stream"] == True - assert span["metadata"]["extra_headers"]["X-Stainless-Helper-Method"] == "chat.completions.stream" + assert "extra_headers" not in span["metadata"] assert span["metadata"]["provider"] == "openai" assert TEST_MODEL in span["metadata"]["model"] assert TEST_PROMPT in str(span["input"]) diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 6348f481..fc8cc7ff 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -12,10 +12,12 @@ from braintrust.integrations.utils import ( _extract_audio_output, _infer_audio_mime_type, + _is_not_given, _materialize_attachment, _parse_openai_usage_metrics, _prettify_response_params, _ResolvedAttachment, + _serialize_response_format, _timing_metrics, _try_to_dict, ) @@ -43,6 +45,143 @@ def start_span(*args, **kwargs): RAW_RESPONSE_HEADER = "x-stainless-raw-response" +# Allowlists filter request/result kwargs into span metadata. Anything outside +# these tuples (extra_headers, user, safety_identifier, ...) is dropped. +_CHAT_COMPLETION_METADATA_PARAMS = ( + "model", + "temperature", + "top_p", + "top_logprobs", + "logprobs", + "n", + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "stop", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "tools", + "tool_choice", + "parallel_tool_calls", + "functions", + "function_call", + "response_format", + "seed", + "modalities", + "audio", + "reasoning_effort", + "service_tier", + "store", + "prediction", + "web_search_options", + "verbosity", + "moderation", +) + +_RESPONSES_METADATA_PARAMS = ( + "model", + "instructions", + "temperature", + "top_p", + "max_output_tokens", + "stream", + "stream_options", + "tools", + "tool_choice", + "parallel_tool_calls", + "reasoning", + "reasoning_effort", + "text", + "text_format", + "response_format", + "include", + "truncation", + "background", + "service_tier", + "store", + "previous_response_id", + "conversation", + "verbosity", + "moderation", +) + +_RESPONSES_RESULT_METADATA_KEYS = ( + "id", + "model", + "status", + "created_at", + "moderation", + "service_tier", + "object", + "background", + "incomplete_details", +) + +_EMBEDDING_METADATA_PARAMS = ( + "model", + "dimensions", + "encoding_format", +) + +_MODERATION_METADATA_PARAMS = ("model",) + +_SPEECH_METADATA_PARAMS = ( + "model", + "voice", + "response_format", + "speed", + "instructions", + "stream_format", +) + +_AUDIO_FILE_METADATA_PARAMS = ( + "model", + "language", + "prompt", + "response_format", + "temperature", + "stream", + "include", + "timestamp_granularities", + "chunking_strategy", +) + +_IMAGE_METADATA_PARAMS = ( + "model", + "n", + "size", + "quality", + "style", + "response_format", + "background", + "moderation", + "output_format", + "output_compression", +) + + +def _get_attr(obj: Any, key: str) -> Any: + if isinstance(obj, dict): + return obj.get(key) + return getattr(obj, key, None) + + +def _filter_metadata(params: dict[str, Any], allowlist: tuple[str, ...]) -> dict[str, Any]: + metadata: dict[str, Any] = {"provider": "openai"} + for key in allowlist: + if key not in params: + continue + value = params[key] + if value is None or _is_not_given(value): + continue + if key == "response_format": + value = _serialize_response_format(value) + metadata[key] = value + return metadata + + class NamedWrapper: def __init__(self, wrapped: Any): self.__wrapped = wrapped @@ -105,10 +244,9 @@ def log_headers(response: Any, span: Span): ) -def _extract_moderation_metadata(response: dict[str, Any]) -> dict[str, Any]: - if "moderation" in response and response.get("moderation") is not None: - return {"moderation": response.get("moderation")} - return {} +def _extract_moderation_metadata(response: Any) -> dict[str, Any]: + moderation = response.get("moderation") if isinstance(response, dict) else getattr(response, "moderation", None) + return {"moderation": moderation} if moderation is not None else {} def _raw_response_requested(kwargs: dict[str, Any]) -> bool: @@ -421,6 +559,14 @@ async def call(): # --------------------------------------------------------------------------- +def _choices_output(response: Any, *, audio_format: str | None) -> Any: + choices = _get_attr(response, "choices") + if choices is None: + return None + dumped = [_try_to_dict(choice) for choice in choices] + return _process_attachments_in_chat_output(dumped, audio_format=audio_format) + + class ChatCompletionWrapper: def __init__(self, create_fn: Callable[..., Any] | None, acreate_fn: Callable[..., Any] | None): self.create_fn = create_fn @@ -471,13 +617,12 @@ def gen(): return _RawResponseWithTracedStream(create_response, _TracedStream(raw_response, gen())) return _TracedStream(raw_response, gen()) else: - log_response = _try_to_dict(raw_response) - metrics = _parse_metrics_from_usage(log_response.get("usage", {})) + metrics = _parse_metrics_from_usage(getattr(raw_response, "usage", None)) metrics["time_to_first_token"] = time.time() - start span.log( metrics=metrics, - output=_process_attachments_in_chat_output(log_response["choices"], audio_format=audio_format), - metadata=_extract_moderation_metadata(log_response), + output=_choices_output(raw_response, audio_format=audio_format), + metadata=_extract_moderation_metadata(raw_response), ) return create_response if (raw_requested and hasattr(create_response, "parse")) else raw_response finally: @@ -532,13 +677,12 @@ async def gen(): return _RawResponseWithTracedStream(create_response, _AsyncTracedStream(raw_response, streamer)) return _AsyncTracedStream(raw_response, streamer) else: - log_response = _try_to_dict(raw_response) - metrics = _parse_metrics_from_usage(log_response.get("usage")) + metrics = _parse_metrics_from_usage(getattr(raw_response, "usage", None)) metrics["time_to_first_token"] = time.time() - start span.log( metrics=metrics, - output=_process_attachments_in_chat_output(log_response["choices"], audio_format=audio_format), - metadata=_extract_moderation_metadata(log_response), + output=_choices_output(raw_response, audio_format=audio_format), + metadata=_extract_moderation_metadata(raw_response), ) return create_response if (raw_requested and hasattr(create_response, "parse")) else raw_response finally: @@ -547,21 +691,13 @@ async def gen(): @classmethod def _parse_params(cls, params: dict[str, Any]) -> dict[str, Any]: - # First, destructively remove span_info ret = params.pop("span_info", {}) - - # Then, copy the rest of the params - params = prettify_params(params) - messages = params.pop("messages", None) - - # Process attachments in input (convert data URLs to Attachment objects) - processed_input = _process_attachments_in_input(messages) - + processed_input = _process_attachments_in_input(params.get("messages")) return merge_dicts( ret, { "input": processed_input, - "metadata": {**params, "provider": "openai"}, + "metadata": _filter_metadata(params, _CHAT_COMPLETION_METADATA_PARAMS), }, ) @@ -1043,8 +1179,7 @@ def gen(): return _RawResponseWithTracedStream(create_response, _TracedStream(raw_response, gen())) return _TracedStream(raw_response, gen()) else: - log_response = _try_to_dict(raw_response) - event_data = self._parse_event_from_result(log_response) + event_data = self._parse_event_from_result(raw_response) if "metrics" not in event_data: event_data["metrics"] = {} event_data["metrics"]["time_to_first_token"] = time.time() - start @@ -1100,8 +1235,7 @@ async def gen(): return _RawResponseWithTracedStream(create_response, _AsyncTracedStream(raw_response, streamer)) return _AsyncTracedStream(raw_response, streamer) else: - log_response = _try_to_dict(raw_response) - event_data = self._parse_event_from_result(log_response) + event_data = self._parse_event_from_result(raw_response) if "metrics" not in event_data: event_data["metrics"] = {} event_data["metrics"]["time_to_first_token"] = time.time() - start @@ -1114,41 +1248,38 @@ async def gen(): @classmethod def _parse_params(cls, params: dict[str, Any]) -> dict[str, Any]: - # First, destructively remove span_info ret = params.pop("span_info", {}) - - # Then, copy the rest of the params - params = prettify_params(params) - input_data = params.pop("input", None) - - # Process attachments in input (convert data URLs to Attachment objects) - processed_input = _process_attachments_in_input(input_data) - + processed_input = _process_attachments_in_input(params.get("input")) return merge_dicts( ret, { "input": processed_input, - "metadata": {**params, "provider": "openai"}, + "metadata": _filter_metadata(params, _RESPONSES_METADATA_PARAMS), }, ) @classmethod - def _parse_event_from_result(cls, result: dict[str, Any]) -> dict[str, Any]: - """Parse event from response result""" - data = {"metrics": {}} + def _parse_event_from_result(cls, result: Any) -> dict[str, Any]: + data: dict[str, Any] = {"metrics": {}} if not result: return data - if "output" in result: - data["output"] = result["output"] + output = _get_attr(result, "output") + if output is not None: + data["output"] = output - metadata = {k: v for k, v in result.items() if k not in ["output", "usage"]} + metadata = {} + for key in _RESPONSES_RESULT_METADATA_KEYS: + value = _get_attr(result, key) + if value is not None: + metadata[key] = value if metadata: data["metadata"] = metadata - if "usage" in result: - data["metrics"] = _parse_metrics_from_usage(result["usage"]) + usage = _get_attr(result, "usage") + if usage is not None: + data["metrics"] = _parse_metrics_from_usage(usage) return data @@ -1363,22 +1494,17 @@ async def acreate(self, *args: Any, **kwargs: Any) -> Any: self.process_output(log_response, span) return raw_response + _metadata_params: tuple[str, ...] = () + @classmethod def _parse_params(cls, params: dict[str, Any]) -> dict[str, Any]: - # First, destructively remove span_info ret = params.pop("span_info", {}) - - params = prettify_params(params) - input_data = params.pop("input", None) - - # Process attachments in input (convert data URLs to Attachment objects) - processed_input = _process_attachments_in_input(input_data) - + processed_input = _process_attachments_in_input(params.get("input")) return merge_dicts( ret, { "input": processed_input, - "metadata": {**params, "provider": "openai"}, + "metadata": _filter_metadata(params, cls._metadata_params), }, ) @@ -1433,10 +1559,9 @@ def process_output(self, response: Any, span: Span): @classmethod def _parse_params(cls, params: dict[str, Any]) -> dict[str, Any]: ret = params.pop("span_info", {}) - params = prettify_params(params) - prompt = params.pop("prompt", None) - image = params.pop("image", None) - mask = params.pop("mask", None) + prompt = params.get("prompt") + image = params.get("image") + mask = params.get("mask") input_data = clean_nones( { @@ -1450,7 +1575,7 @@ def _parse_params(cls, params: dict[str, Any]) -> dict[str, Any]: ret, { "input": prompt if (prompt is not None and len(input_data) == 1) else (input_data or None), - "metadata": {**params, "provider": "openai"}, + "metadata": _filter_metadata(params, _IMAGE_METADATA_PARAMS), }, ) @@ -1471,6 +1596,8 @@ def __init__(self, create_fn: Callable[..., Any] | None, acreate_fn: Callable[.. class EmbeddingWrapper(BaseWrapper): + _metadata_params = _EMBEDDING_METADATA_PARAMS + def __init__(self, create_fn: Callable[..., Any] | None, acreate_fn: Callable[..., Any] | None): super().__init__(create_fn, acreate_fn, "Embedding") @@ -1486,6 +1613,8 @@ def process_output(self, response: dict[str, Any], span: Span): class ModerationWrapper(BaseWrapper): + _metadata_params = _MODERATION_METADATA_PARAMS + def __init__(self, create_fn: Callable[..., Any] | None, acreate_fn: Callable[..., Any] | None): super().__init__(create_fn, acreate_fn, "Moderation") @@ -1496,6 +1625,8 @@ def process_output(self, response: Any, span: Span): class SpeechWrapper(BaseWrapper): + _metadata_params = _SPEECH_METADATA_PARAMS + def __init__(self, create_fn: Callable[..., Any] | None, acreate_fn: Callable[..., Any] | None): super().__init__(create_fn, acreate_fn, "Speech") @@ -1506,19 +1637,18 @@ def process_output(self, response: Any, span: Span): class _AudioFileWrapper(BaseWrapper): """Base for transcription/translation wrappers that accept audio file input.""" + _metadata_params = _AUDIO_FILE_METADATA_PARAMS + @classmethod def _parse_params(cls, params: dict[str, Any]) -> dict[str, Any]: ret = params.pop("span_info", {}) - params = prettify_params(params) - # Remove the file object after prettifying — prettify_params already - # made a copy so the original kwargs (used by the API call) are preserved. - file_input = _materialize_logged_file_input(params.pop("file", None)) + file_input = _materialize_logged_file_input(params.get("file")) input_data = {"file": file_input} if file_input is not None else None return merge_dicts( ret, { "input": input_data, - "metadata": {**params, "provider": "openai"}, + "metadata": _filter_metadata(params, cls._metadata_params), }, ) From 08442129c9f302b208e109d7325a6c26b5ee0402 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Tue, 21 Jul 2026 17:06:34 +0000 Subject: [PATCH 2/3] Drop _get_attr helper and redundant None check in _filter_metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `_get_attr` only earned its keep in `_extract_moderation_metadata` (which legitimately sees both dicts from stream postprocess and pydantic objects from non-streaming). Its other callers — `_parse_event_from_result` and `_choices_output` — only ever see pydantic responses from the OpenAI SDK, so plain `getattr(x, k, None)` suffices. - `_filter_metadata` was checking `value is None or _is_not_given(value)` — the `None` half was redundant defensive belt: `_is_not_given` already handles the sentinels, and the old denylist behavior kept None-valued params too. Co-Authored-By: Claude Opus 4.7 --- py/src/braintrust/integrations/openai/tracing.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index fc8cc7ff..f13eecad 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -162,19 +162,13 @@ def start_span(*args, **kwargs): ) -def _get_attr(obj: Any, key: str) -> Any: - if isinstance(obj, dict): - return obj.get(key) - return getattr(obj, key, None) - - def _filter_metadata(params: dict[str, Any], allowlist: tuple[str, ...]) -> dict[str, Any]: metadata: dict[str, Any] = {"provider": "openai"} for key in allowlist: if key not in params: continue value = params[key] - if value is None or _is_not_given(value): + if _is_not_given(value): continue if key == "response_format": value = _serialize_response_format(value) @@ -560,7 +554,7 @@ async def call(): def _choices_output(response: Any, *, audio_format: str | None) -> Any: - choices = _get_attr(response, "choices") + choices = getattr(response, "choices", None) if choices is None: return None dumped = [_try_to_dict(choice) for choice in choices] @@ -1265,19 +1259,19 @@ def _parse_event_from_result(cls, result: Any) -> dict[str, Any]: if not result: return data - output = _get_attr(result, "output") + output = getattr(result, "output", None) if output is not None: data["output"] = output metadata = {} for key in _RESPONSES_RESULT_METADATA_KEYS: - value = _get_attr(result, key) + value = getattr(result, key, None) if value is not None: metadata[key] = value if metadata: data["metadata"] = metadata - usage = _get_attr(result, "usage") + usage = getattr(result, "usage", None) if usage is not None: data["metrics"] = _parse_metrics_from_usage(usage) From 3e5412c85308922a58031f23295572e5d288ad74 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Tue, 21 Jul 2026 17:16:59 +0000 Subject: [PATCH 3/3] Serialize text_format the same way as response_format `text_format` on Responses API (`.parse(text_format=SomeModel)`) accepts a Pydantic BaseModel class exactly like `response_format` on Chat Completions. It was allowlisted but not passed through `_serialize_response_format`, so the class reference landed in span metadata as `` instead of the JSON schema. Co-Authored-By: Claude Opus 4.7 --- py/src/braintrust/integrations/openai/tracing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index f13eecad..47744247 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -170,7 +170,7 @@ def _filter_metadata(params: dict[str, Any], allowlist: tuple[str, ...]) -> dict value = params[key] if _is_not_given(value): continue - if key == "response_format": + if key in ("response_format", "text_format"): value = _serialize_response_format(value) metadata[key] = value return metadata