diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 099be74515..fd98657da7 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -1476,12 +1476,52 @@ def _extract_http_error_served_model(exc: urllib.error.HTTPError) -> str | None: return model if isinstance(model, str) else None -def _format_gateway_error_telemetry(telemetry: dict[str, str | int]) -> str: +def _extract_success_route_telemetry(raw: str) -> dict[str, str | int]: + """Report only bounded structured-route facts from a served gateway response.""" + try: + payload = json.loads(raw) + except (TypeError, ValueError, RecursionError): + return {} + if not isinstance(payload, dict): + return {} + orchestration = payload.get("orchestration") + route = orchestration.get("route") if isinstance(orchestration, dict) else None + if not isinstance(route, dict) or route.get("terminal_reason") != "served": + return {} + stage = route.get("stage") + attempts = route.get("attempted") + if stage not in ("structured_synthesis", "structured_repair") or not ( + isinstance(attempts, list) and 1 <= len(attempts) <= 64 + ): + return {} + if any(not isinstance(attempt, dict) for attempt in attempts): + return {} + if attempts[-1].get("outcome") != "served" or any( + attempt.get("outcome") not in ( + "request_too_large", "retryable_transport", "deadline_exceeded", "fail_closed" + ) for attempt in attempts[:-1] + ): + return {} + recovered = any( + type(attempt.get("provider_status")) is int + and attempt["provider_status"] == 429 + for attempt in attempts[:-1] + ) + return { + "provider_attempt_count": len(attempts), + "route_stage": stage, + "route_outcome": "served", + "rate_limit_recovered": int(recovered), + } + + +def _format_gateway_telemetry(telemetry: dict[str, str | int]) -> str: """Format only allowlisted scalar receipt fields for a public Actions log.""" ordered_keys = ( "provider_attempt_count", "route_stage", "route_outcome", + "rate_limit_recovered", "provider_name", "upstream_phase", "attempt_number", @@ -1749,7 +1789,7 @@ def call_llm( elapsed = time.monotonic() - attempt_started current_failure = _stable_failure_diagnostic(exc) model_note = served_model or "unknown" - gateway_note = _format_gateway_error_telemetry(gateway_telemetry) + gateway_note = _format_gateway_telemetry(gateway_telemetry) capacity_unavailable = is_provider_capacity_http_status(http_status) capacity_note = ( " outcome=provider_capacity_unavailable" @@ -1793,10 +1833,12 @@ def call_llm( f"Noema review failed closed: {current_failure}{suffix}" ) from exc elapsed = time.monotonic() - attempt_started + gateway_note = _format_gateway_telemetry(_extract_success_route_telemetry(raw)) print( f"::notice::Noema gateway attempt outcome=success phase={active_phase} " f"duration={elapsed:.1f}s served_model={served_model or 'unknown'}; " "caller attempts=1." + + (f" gateway {gateway_note}" if gateway_note else "") ) return verdict diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index cc048372d7..3f7b45e57e 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1667,7 +1667,7 @@ def test_structured_route_receipt_exports_only_bounded_scalars(): error = noema.urllib.error.HTTPError("https://llm.example.test", 429, "", {}, io.BytesIO(body)) telemetry = noema._extract_http_error_telemetry(error) - rendered = noema._format_gateway_error_telemetry(telemetry) + rendered = noema._format_gateway_telemetry(telemetry) assert rendered == ( "provider_attempt_count=2 route_stage=structured_synthesis " "route_outcome=fail_closed upstream_status=429" @@ -1688,6 +1688,46 @@ def test_malformed_structured_route_fails_closed_without_legacy_attempt_fallback assert noema._extract_http_error_telemetry(error) == {} +def test_served_structured_route_reports_429_recovery_without_route_secrets(monkeypatch, capsys): + secret = "never-print-route-identity" + monkeypatch.setenv("NOEMA_LLM_API_URL", "https://llm.example.test/chat") + monkeypatch.setenv("NOEMA_LLM_API_KEY", "secret") + monkeypatch.setattr(noema, "validate_substantive_verdict", lambda *_args: None) + payload = { + "choices": [{"message": {"content": json.dumps({ + "decision": "comment", "summary": "review complete", "findings": [], + })}}], + "orchestration": {"route": { + "stage": "structured_synthesis", + "terminal_reason": "served", + "attempted": [ + {"agent_id": secret, "outcome": "retryable_transport", "provider_status": 429}, + {"agent_id": secret, "outcome": "served"}, + ], + "eligible_agent_ids": [secret], + }}, + } + + class Opener: + def open(self, _request): + return FakeResponse(payload) + + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *_args: Opener()) + assert noema.call_llm("owner/repo", 1, make_pr(), "diff", False, "head")["decision"] == "comment" + output = capsys.readouterr().out + assert "duration=" in output + assert ( + "gateway provider_attempt_count=2 route_stage=structured_synthesis " + "route_outcome=served rate_limit_recovered=1" + ) in output + assert secret not in output + + payload["orchestration"]["route"]["attempted"][0]["provider_status"] = True + assert noema._extract_success_route_telemetry(json.dumps(payload))["rate_limit_recovered"] == 0 + payload["orchestration"]["route"]["attempted"].append({"outcome": secret}) + assert noema._extract_success_route_telemetry(json.dumps(payload)) == {} + + def test_is_provider_capacity_http_status_covers_only_capacity_class(): """429/5xx are capacity; other statuses stay ordinary transport failures.""" assert noema.is_provider_capacity_http_status(429) is True