From 26a84cd0a5980b2b7be1640c2565a5c28eb3b479 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 25 Sep 2026 22:12:04 +0900 Subject: [PATCH 1/3] fix(review): advance gateway pin for 429 recovery --- ...03-contextual-orchestrator-vendored-free-zdr.md | 14 +++++++++++++- .../ci/contextual_orchestrator_review_sidecar.sh | 2 +- ...textual_orchestrator_review_sidecar_contract.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 6629675f14..9d656e107c 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -24,7 +24,7 @@ all five, and auto-optimize routing by cost. 1. **Vendoring, pinned**: `scripts/ci/contextual_orchestrator_review_sidecar.sh` clones `ContextualWisdomLab/contextual-orchestrator` at an exact SHA - (`767e67fbc6b881a452761f32abb69b9971b9b03b` today) into `RUNNER_TEMP`. The + (`5665b0ad1e07ffb5e9f8c59e44b6b2a785298013` today) into `RUNNER_TEMP`. The source's `requirements.lock` is installed with `--require-hashes` and `--no-deps`, so dependency resolution cannot silently move the reviewed runtime. @@ -294,3 +294,15 @@ all five, and auto-optimize routing by cost. per-agent attempt; it changes only *which* agent gets tried next, never any per-attempt timeout, consistent with the 2026-08-31 amendment above. No other contextual-orchestrator behavior changes with this pin advance. +- **Proposed 2026-09-25 amendment: admit gateway-owned recovery from a free-pool 429 + storm.** The vendored pin advances from + `767e67fbc6b881a452761f32abb69b9971b9b03b` to merged main commit + `5665b0ad1e07ffb5e9f8c59e44b6b2a785298013`. The old pin predates the + rate-limit-aware admission path; Noema run 36024200990 ended with HTTP 429 + after one caller request. That caller count does not reveal internal provider + attempts. The new pin includes bounded cooldown and distinct-candidate + recovery for virtual `orchestrator/free` requests. Its offline storm suite + passes 23 cases; this is not evidence that the live provider pool can always + recover or that Noema has approved any PR. Explicit-model and unknown-outcome + no-replay rules still apply. The source commit's push checks are failing at + the time of this proposal, so this pin is not release or production proof. diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index 3c2a1b51b9..5abcada3e0 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -14,7 +14,7 @@ # (fail-closed zero-cost) pool. set -euo pipefail -ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-767e67fbc6b881a452761f32abb69b9971b9b03b}" +ORCHESTRATOR_PIN_SHA="${ORCHESTRATOR_PIN_SHA:-5665b0ad1e07ffb5e9f8c59e44b6b2a785298013}" ORCHESTRATOR_GIT_URL="${ORCHESTRATOR_GIT_URL:-https://github.com/ContextualWisdomLab/contextual-orchestrator.git}" # The Strix gate and Noema SSRF guard accept this one process-local origin. # Keep it fixed so an environment override cannot create an unvalidated sidecar. diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index b279d33a98..9628e2559e 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -40,7 +40,7 @@ ) GATEWAY_MODEL = "contextual-orchestrator/orchestrator/free" -ORCH_PIN_SHA = "767e67fbc6b881a452761f32abb69b9971b9b03b" +ORCH_PIN_SHA = "5665b0ad1e07ffb5e9f8c59e44b6b2a785298013" def _read(path: Path) -> str: From 7bcd7048293c732d4de2c84a9fe9ef611a3a534f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 26 Sep 2026 02:19:44 +0900 Subject: [PATCH 2/3] fix(noema): allowlist structured route failure receipt --- scripts/ci/noema_review_gate.py | 23 ++++++++++++++++++- tests/test_noema_review_gate.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index c8709304fc..5379c77a0e 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -1463,11 +1463,30 @@ def _extract_http_error_telemetry(exc: urllib.error.HTTPError) -> dict[str, str telemetry: dict[str, str | int] = {} model = _safe_model_identifier(detail.get("model")) terminal_reason = _safe_model_identifier(detail.get("terminal_reason")) - attempts = detail.get("attempts") + route = detail.get("route") + attempts = detail.get("attempts") if "route" not in detail else None if model is not None: telemetry["served_model"] = model if terminal_reason is not None: telemetry["terminal_reason"] = terminal_reason + if isinstance(route, dict): + stage = route.get("stage") + if stage in ("conduct", "structured_synthesis", "structured_repair"): + telemetry["route_stage"] = stage + route_attempts = route.get("attempted") + if isinstance(route_attempts, list) and 0 < len(route_attempts) <= 64: + telemetry["provider_attempt_count"] = len(route_attempts) + last_route_attempt = route_attempts[-1] + if isinstance(last_route_attempt, dict): + outcome = last_route_attempt.get("outcome") + if outcome in ( + "request_too_large", "retryable_transport", + "deadline_exceeded", "fail_closed", + ): + telemetry["route_outcome"] = outcome + provider_status = last_route_attempt.get("provider_status") + if type(provider_status) is int and 100 <= provider_status <= 599: + telemetry["upstream_status"] = provider_status if isinstance(attempts, list) and attempts and len(attempts) <= 64: telemetry["provider_attempt_count"] = len(attempts) last_attempt = attempts[-1] @@ -1497,6 +1516,8 @@ def _format_gateway_error_telemetry(telemetry: dict[str, str | int]) -> str: """Format only allowlisted scalar receipt fields for a public Actions log.""" ordered_keys = ( "provider_attempt_count", + "route_stage", + "route_outcome", "provider_name", "upstream_phase", "attempt_number", diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 5053645802..0b4a456485 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1649,6 +1649,45 @@ def open(self, request): assert secret not in diagnostic +def test_structured_route_receipt_exports_only_bounded_scalars(): + secret = "never-print-provider-payload" + body = json.dumps({"error": {"detail": { + "route": { + "stage": "structured_synthesis", + "attempted": [ + {"agent_id": secret, "model": secret, "outcome": "retryable_transport", + "provider_status": None, "message": secret}, + {"agent_id": secret, "model": secret, "outcome": "fail_closed", + "provider_status": 429, "message": secret}, + ], + "eligible_agent_ids": [secret], + }, + "attempts": [{"provider_name": secret}], + }}}).encode() + error = noema.urllib.error.HTTPError("https://llm.example.test", 429, "", {}, io.BytesIO(body)) + + telemetry = noema._extract_http_error_telemetry(error) + rendered = noema._format_gateway_error_telemetry(telemetry) + assert rendered == ( + "provider_attempt_count=2 route_stage=structured_synthesis " + "route_outcome=fail_closed upstream_status=429" + ) + assert secret not in rendered + + +def test_malformed_structured_route_fails_closed_without_legacy_attempt_fallback(): + secret = "never-print-provider-payload" + body = json.dumps({"error": {"detail": { + "route": {"stage": secret, "attempted": [ + {"outcome": secret, "provider_status": True} + ] * 65}, + "attempts": [{"provider_name": secret}], + }}}).encode() + error = noema.urllib.error.HTTPError("https://llm.example.test", 502, "", {}, io.BytesIO(body)) + + assert noema._extract_http_error_telemetry(error) == {} + + def test_is_provider_capacity_http_status_covers_only_capacity_class(): """429/5xx are capacity; other statuses stay ordinary transport failures.""" assert noema.is_provider_capacity_http_status(429) is True From 3805e2a0d3dc9608040b14efbd56c07d178bba14 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 29 Sep 2026 02:40:31 +0900 Subject: [PATCH 3/3] test(noema): cover unrecognized structured route receipts Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015JrSXPN3cJiRFyeJ3UTFT7 --- tests/test_noema_review_gate.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index cc048372d7..2097939812 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1688,6 +1688,25 @@ def test_malformed_structured_route_fails_closed_without_legacy_attempt_fallback assert noema._extract_http_error_telemetry(error) == {} +@pytest.mark.parametrize( + "last_attempt", + [ + "not-a-receipt", + {"outcome": "never-print-provider-payload", "provider_status": 99}, + ], +) +def test_structured_route_counts_attempts_but_drops_unrecognized_last_receipt(last_attempt): + body = json.dumps({"error": {"detail": {"route": { + "stage": "conduct", "attempted": [last_attempt], + }}}}).encode() + error = noema.urllib.error.HTTPError("https://llm.example.test", 502, "", {}, io.BytesIO(body)) + + assert noema._extract_http_error_telemetry(error) == { + "route_stage": "conduct", + "provider_attempt_count": 1, + } + + def test_is_provider_capacity_http_status_covers_only_capacity_class(): """429/5xx are capacity; other statuses stay ordinary transport failures.""" assert noema.is_provider_capacity_http_status(429) is True