Feat/referee self inconsistency floor

benchmaxxing/extract.py

@@ -87,6 +87,51 @@ def is_abstention(text: str) -> bool:
 )
 
 
+def declared_mcq_choice(
+    text: str,
+    options: tuple[str, ...] | list[str],
+) -> tuple[str, bool]:
+    """Return (declared_answer, declared_flag).
+
+    Unlike parse_mcq_choice(), this only recognizes explicit answer
+    declarations and intentionally ignores fallback heuristics such as
+    trailing-letter scans or last-mentioned-option resolution.
+    """
+
+    if not text:
+        return "", False
+
+    num_options = len(options)
+    if num_options == 0:
+        return "", False
+
+    valid_letters = {
+        chr(ord("A") + i)
+        for i in range(min(num_options, 26))
+    }
+
+    declarations = [
+        _first_group(m)
+        for m in _DECLARATION_REGEX.finditer(text)
+        if _first_group(m) in valid_letters
+    ]
+
+    if declarations:
+        letter = declarations[-1]
+        return options[ord(letter) - ord("A")], True
+
+    stripped = text.strip()
+
+    if (
+        len(stripped) == 1
+        and stripped.upper() in valid_letters
+    ):
+        return options[ord(stripped.upper()) - ord("A")], True
+
+    return "", False
+
+
+
 def _first_group(m: re.Match) -> str:
     """Return the first non-None group from a regex match with alternations."""
     for g in m.groups():

benchmaxxing/gateway.py

@@ -141,7 +141,9 @@ def __init__(
         timeout: float | None = 60.0,
     ):
         self.model = model
-        self.default_decoding = dict(default_decoding or {})
+        self.default_decoding = {"max_output_tokens": 16384}
+        if default_decoding:
+            self.default_decoding.update(default_decoding)
         self.timeout = timeout
         if client is not None:
             # Injected client (used by tests): no SDK import required.

experiments/referee/referee_self_inconsistency.py

@@ -0,0 +1,150 @@
+"""Referee self-inconsistency floor (#417).
+
+Measures whether identical cache-bypassed temperature-0 private re-queries
+produce different answers in the absence of committee influence.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from benchmaxxing.data import load_cases
+from benchmaxxing.extract import parse_legacy_string, declared_mcq_choice
+from experiments.referee.referee_threshold import (
+    _Cache,
+    _key,
+    _mcq,
+    HOLDOUT,
+)
+
+
+
+def build_row(case_id, answer_1, answer_2, declared_1, declared_2):
+    return {
+        "case_id": case_id,
+        "answer_1": answer_1,
+        "answer_2": answer_2,
+        "declared_1": declared_1,
+        "declared_2": declared_2,
+        "temp0_flip": answer_1 != answer_2,
+    }
+
+
+def run_one(case, cache):
+    opts = list(case.options)
+    prompt, _ = _mcq(case)
+
+    raw_1 = cache.complete(
+        HOLDOUT, prompt, temperature=0.0, draw=1
+    )
+    raw_2 = cache.complete(
+        HOLDOUT, prompt, temperature=0.0, draw=2
+    )
+
+    answer_1 = parse_legacy_string(raw_1, opts)
+    answer_2 = parse_legacy_string(raw_2, opts)
+
+    _, declared_1 = declared_mcq_choice(raw_1, opts)
+    _, declared_2 = declared_mcq_choice(raw_2, opts)
+
+    return build_row(
+        case.case_id,
+        answer_1,
+        answer_2,
+        declared_1,
+        declared_2,
+    )
+
+
+def summarize(rows):
+    n = len(rows)
+
+    declared_pairs = sum(
+        1
+        for r in rows
+        if r["declared_1"] and r["declared_2"]
+    )
+
+    undeclared_pairs = sum(
+        1
+        for r in rows
+        if not (r["declared_1"] and r["declared_2"])
+    )
+
+    undeclared_draws = sum(
+        int(not r["declared_1"]) + int(not r["declared_2"])
+        for r in rows
+    )
+
+    unstable = sum(
+        1
+        for r in rows
+        if r["declared_1"]
+        and r["declared_2"]
+        and r["temp0_flip"]
+    )
+
+    stable = declared_pairs - unstable
+
+    return {
+        "n": n,
+        "temperature": 0,
+        "cache_bypassed": True,
+        "declared_pairs": declared_pairs,
+        "undeclared_pairs": undeclared_pairs,
+        "undeclared_draws": undeclared_draws,
+        "stable_cases": stable,
+        "unstable_cases": unstable,
+        "temp0_self_inconsistency_rate": (
+            unstable / declared_pairs
+            if declared_pairs
+            else None
+        ),
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Referee self-inconsistency floor (#417)."
+    )
+    ap.add_argument("--manifest", required=True)
+    ap.add_argument(
+        "--cache",
+        default="experiments/referee/results/referee_self_inconsistency_cache.jsonl",
+    )
+    ap.add_argument(
+        "--out",
+        default="experiments/referee/results",
+    )
+    ap.add_argument("--n", type=int, default=40)
+
+    args = ap.parse_args()
+
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+
+    cache = _Cache(args.cache, _key())
+
+    rows = [
+        run_one(case, cache)
+        for case in load_cases(args.manifest)[:args.n]
+    ]
+
+    summary = summarize(rows)
+    summary["new_api_calls_this_run"] = cache.calls
+
+    (out / "referee_self_inconsistency.jsonl").write_text(
+        "".join(json.dumps(r) + "\n" for r in rows)
+    )
+
+    (out / "referee_self_inconsistency_summary.json").write_text(
+        json.dumps(summary, indent=2)
+    )
+
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

tests/test_extract.py

@@ -169,3 +169,29 @@ def test_leading_article_a_is_not_the_answer(self):
         text = (r"A patient with these findings is best treated with option C. "
                 r"The final answer is $\boxed{C}$")
         assert parse_mcq_choice(text, self.OPTS) == 2  # C, never the article-A (0)
+
+
+
+# ── Declaration detection (Phase 4B) ───────────────────────────────────
+
+def test_declared_mcq_choice_explicit_declaration():
+    from benchmaxxing.extract import declared_mcq_choice
+
+    assert declared_mcq_choice("The answer is B.", OPTS_5) == ("B", True)
+
+
+def test_declared_mcq_choice_bare_letter():
+    from benchmaxxing.extract import declared_mcq_choice
+
+    assert declared_mcq_choice("C", OPTS_5) == ("C", True)
+
+
+def test_declared_mcq_choice_undeclared_reasoning():
+    from benchmaxxing.extract import declared_mcq_choice
+
+    text = (
+        "A is unlikely. B is possible. "
+        "C could fit. D is also plausible."
+    )
+
+    assert declared_mcq_choice(text, OPTS_5) == ("", False)

tests/test_gateway.py

@@ -193,15 +193,15 @@ def test_gemini_backend_injected_client_multimodal_path():
     be.complete("describe", image=b"imgbytes")
     _, contents, kwargs = client.models.received[0]
     assert contents == ["describe", b"imgbytes"]
-    assert "config" not in kwargs  # no decoding overrides -> no config passed
+    assert kwargs["config"] == {"max_output_tokens": 16384}
 
 
 def test_gemini_backend_merges_default_decoding():
     client = _FakeClient()
     be = gateway.GeminiBackend(client=client, default_decoding={"temperature": 0.7, "top_p": 0.9})
     be.complete("hi", decoding={"temperature": 0.1})
     _, _, kwargs = client.models.received[0]
-    assert kwargs["config"] == {"temperature": 0.1, "top_p": 0.9}  # per-call overrides default
+    assert kwargs["config"] == {"temperature": 0.1, "top_p": 0.9, "max_output_tokens": 16384}  # per-call overrides default
 
 
 def test_timeout_ms_converts_seconds_and_rejects_non_positive():
@@ -264,3 +264,16 @@ def test_gemini_backend_leaves_http_options_unset_when_timeout_disabled(monkeypa
     captured = _stub_genai(monkeypatch)
     gateway.GeminiBackend(api_key="k", timeout=None)
     assert captured["http_options"] is None
+
+
+
+def test_gemini_backend_applies_default_max_output_tokens():
+    client = _FakeClient()
+
+    be = gateway.GeminiBackend(client=client)
+
+    be.complete("hi")
+
+    _, _, kwargs = client.models.received[0]
+
+    assert kwargs["config"]["max_output_tokens"] == 16384