fix(judge): give a judge outage its own outcome, not JUDGE MISMATCH

CHANGELOG.md

@@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 ### Changed
 - The `claw-eval` port now uses the same `test-cases/<suite>/<task-identifier>/task.json` layout as the native corpora, instead of flat `<task-identifier>.json` files. Case discovery in `clawbench-batch` and the TUI is a plain `*/task.json` search again, and the `validate-task` workflow covers the suite without special-casing.
 
+### Fixed
+- Fixed a judge-provider outage (or an unparseable judge reply) being recorded as an agent failure. `run.py` now exits 3 instead of 1 when the judge never renders a verdict, `batch.py` gives it its own `judge_inconclusive` bucket in `batch-summary.json` instead of folding it into `failed`, and `clawbench-rescore` now retries a cached `match: null` verdict even without `--force`.
+
 ## [0.10.0] - 2026-08-30
 ### Added
 - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia).

src/clawbench/eval/rescore.py

@@ -74,8 +74,11 @@ def rescore_one(
         judge_p = run_dir / JUDGE_FILE[rubric]
         if judge_p.exists() and not force:
             try:
-                out[rubric] = json.loads(judge_p.read_text())
-                continue
+                cached = json.loads(judge_p.read_text())
+                # A cached match=None is not a scored result: retry it.
+                if cached.get("match") is not None:
+                    out[rubric] = cached
+                    continue
             except Exception:
                 pass
         verdict = judge_funcs[rubric](model_cfg, judge_model, instruction, intercept)
@@ -280,7 +283,10 @@ def main() -> int:
             if not m.get("intercepted"):
                 continue
             needs = any(
-                args.force or not (rd / JUDGE_FILE[r]).exists() for r in rubrics
+                args.force
+                or not (rd / JUDGE_FILE[r]).exists()
+                or json.loads((rd / JUDGE_FILE[r]).read_text()).get("match") is None
+                for r in rubrics
             )
             if needs:
                 pending.append(rd)

src/clawbench/runner/batch.py

@@ -316,6 +316,11 @@ async def run_job(
                     job.status = "passed"
                 elif proc.returncode == 1:
                     job.status = "failed"
+                elif proc.returncode == 3:
+                    # run.py's own signal for "judge never rendered a verdict":
+                    # kept out of "failed" so a judge outage cannot masquerade
+                    # as the agent having failed the task.
+                    job.status = "judge_inconclusive"
                 else:
                     job.status = "error"
             except asyncio.CancelledError:
@@ -352,7 +357,7 @@ async def run_job(
         # Task cancelled while waiting on semaphore, throttle wait, or
         # before subprocess was created.  "running" can appear here if
         # CancelledError hit after status was set but before proc started.
-        if job.status not in ("passed", "failed", "error"):
+        if job.status not in ("passed", "failed", "error", "judge_inconclusive"):
             job.status = "skipped"
         raise
 
@@ -362,10 +367,12 @@ def print_progress(jobs: list[Job], start: float) -> None:
     running = sum(1 for j in jobs if j.status == "running")
     passed = sum(1 for j in jobs if j.status == "passed")
     failed = sum(1 for j in jobs if j.status in ("failed", "error"))
+    inconclusive = sum(1 for j in jobs if j.status == "judge_inconclusive")
     elapsed = fmt_duration(time.monotonic() - start)
     print(
         f"[{ts()}] [BATCH] {done}/{len(jobs)} done | {running} running | "
-        f"{passed} passed, {failed} failed | {elapsed} elapsed",
+        f"{passed} passed, {failed} failed, {inconclusive} judge-inconclusive | "
+        f"{elapsed} elapsed",
         file=sys.stderr,
     )
 
@@ -401,7 +408,7 @@ def print_summary(
         totals[j.status] = totals.get(j.status, 0) + 1
     parts = [
         f"{totals.get(s, 0)} {s}"
-        for s in ("passed", "failed", "error", "skipped")
+        for s in ("passed", "failed", "error", "judge_inconclusive", "skipped")
         if totals.get(s)
     ]
     print(f"\nTotal: {len(jobs)} jobs | {' | '.join(parts)}")
@@ -566,7 +573,7 @@ def write_summary_json(
         ],
         "totals": {
             s: sum(1 for j in jobs if j.status == s)
-            for s in ("passed", "failed", "error", "skipped")
+            for s in ("passed", "failed", "error", "judge_inconclusive", "skipped")
         },
     }
     (base_output / "batch-summary.json").write_text(json.dumps(data, indent=2))

src/clawbench/runner/run.py

@@ -794,7 +794,10 @@ def handle_sigint(sig, frame):
             f"\nINTERCEPTED but JUDGE {'MISMATCH' if verdict is False else 'INCONCLUSIVE'} "
             f"— results in {output_dir}\n  reason: {reason[:200]}"
         )
-        sys.exit(1)
+        # match=None means the judge never rendered a verdict (outage, retries
+        # exhausted, unparseable reply): give it its own exit code instead of
+        # sharing exit 1 with a genuine JUDGE MISMATCH.
+        sys.exit(1 if verdict is False else 3)
     if final_pass:
         status = "INTERCEPTED" if args.no_judge else "INTERCEPTED + JUDGE MATCH"
         print(f"\n{status} — results in {output_dir}")

tests/test_batch_judge_inconclusive.py

@@ -0,0 +1,99 @@
+"""Regression tests for issue #299: a judge outage must not be recorded as an
+agent failure.
+
+run.py now exits 3, instead of sharing exit 1 with a genuine JUDGE MISMATCH,
+when the judge never renders a verdict (match=None). These tests drive
+batch.py's run_job() against a mocked subprocess to lock in that exit 3 lands
+in its own `judge_inconclusive` bucket, kept out of `failed`/`error`, all the
+way into batch-summary.json's totals.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+
+import pytest
+
+from clawbench.runner import batch
+from clawbench.runner.batch import Job, StartupThrottle, run_job, write_summary_json
+
+
+class _FakeProc:
+    def __init__(self, returncode: int) -> None:
+        self.returncode = returncode
+        self.pid = 99999
+
+    async def communicate(self):
+        return b"", None
+
+
+def _run_one_job(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, returncode: int
+) -> Job:
+    batch.shutdown_event = asyncio.Event()
+    batch.running_procs.clear()
+
+    async def fake_create_subprocess_exec(*args, **kwargs):
+        return _FakeProc(returncode)
+
+    monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
+
+    job = Job(model="model-a", case_dir=tmp_path / "case", case_name="case")
+    log_dir = tmp_path / "batch-logs"
+    log_dir.mkdir()
+
+    asyncio.run(
+        run_job(
+            job,
+            asyncio.Semaphore(1),
+            StartupThrottle(0),
+            tmp_path,
+            log_dir,
+            [job],
+            0.0,
+            no_upload=True,
+        )
+    )
+    return job
+
+
+@pytest.mark.parametrize(
+    "returncode,expected_status",
+    [(0, "passed"), (1, "failed"), (3, "judge_inconclusive"), (2, "error")],
+)
+def test_run_job_maps_exit_code_to_status(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    returncode: int,
+    expected_status: str,
+) -> None:
+    job = _run_one_job(monkeypatch, tmp_path, returncode)
+    assert job.status == expected_status
+
+
+def test_judge_inconclusive_counted_separately_in_batch_summary_json(
+    tmp_path: Path,
+) -> None:
+    jobs = [
+        Job(model="m", case_dir=tmp_path, case_name="passed-case", status="passed"),
+        Job(model="m", case_dir=tmp_path, case_name="failed-case", status="failed"),
+        Job(
+            model="m",
+            case_dir=tmp_path,
+            case_name="inconclusive-case",
+            status="judge_inconclusive",
+        ),
+    ]
+
+    write_summary_json(jobs, tmp_path, elapsed=1.0, max_concurrent=1, started_at="now")
+
+    data = json.loads((tmp_path / "batch-summary.json").read_text())
+    assert data["totals"] == {
+        "passed": 1,
+        "failed": 1,
+        "error": 0,
+        "judge_inconclusive": 1,
+        "skipped": 0,
+    }

tests/test_rescore_retarget.py

@@ -0,0 +1,75 @@
+"""Regression test for issue #299, ask 3: clawbench-rescore must retarget a
+run whose cached judge verdict is `match: null` (the judge never rendered a
+verdict last time) even without --force, since that cached file is not a
+scored result: it is the same outage this issue is about, just replayed
+from disk instead of from a live judge call.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from clawbench.eval.rescore import rescore_one
+
+
+def _make_run_dir(tmp_path: Path) -> Path:
+    run_dir = tmp_path / "model-a" / "run-1"
+    (run_dir / "data").mkdir(parents=True)
+    (run_dir / "run-meta.json").write_text(
+        json.dumps({"intercepted": True, "instruction": "do the task"})
+    )
+    (run_dir / "data" / "interception.json").write_text(
+        json.dumps({"request": {"url": "https://example.test"}})
+    )
+    return run_dir
+
+
+def test_cached_inconclusive_verdict_is_retried_without_force(tmp_path: Path) -> None:
+    run_dir = _make_run_dir(tmp_path)
+    (run_dir / "judge.json").write_text(
+        json.dumps({"match": None, "reason": "judge_call_failed: timeout"})
+    )
+
+    calls = []
+
+    def fake_judge(model_cfg, judge_model, instruction, intercept):
+        calls.append(1)
+        return {"match": True, "reason": "fulfills it"}
+
+    out = rescore_one(
+        model_cfg={},
+        judge_model="judge-a",
+        run_dir=run_dir,
+        force=False,
+        rubrics=["strict"],
+        judge_funcs={"strict": fake_judge},
+    )
+
+    assert len(calls) == 1  # retried despite force=False
+    assert out["strict"]["match"] is True
+
+
+def test_cached_scored_verdict_is_not_retried_without_force(tmp_path: Path) -> None:
+    run_dir = _make_run_dir(tmp_path)
+    (run_dir / "judge.json").write_text(
+        json.dumps({"match": False, "reason": "did not fulfill it"})
+    )
+
+    calls = []
+
+    def fake_judge(model_cfg, judge_model, instruction, intercept):
+        calls.append(1)
+        return {"match": True, "reason": "should not be reached"}
+
+    out = rescore_one(
+        model_cfg={},
+        judge_model="judge-a",
+        run_dir=run_dir,
+        force=False,
+        rubrics=["strict"],
+        judge_funcs={"strict": fake_judge},
+    )
+
+    assert calls == []  # cached scored verdict wins, no retry
+    assert out["strict"]["match"] is False

tests/test_run_judge_stage.py

@@ -237,8 +237,9 @@ def explode(*args: Any, **kwargs: Any) -> dict:
     with pytest.raises(SystemExit) as excinfo:
         run_mod.main()
 
-    # Inconclusive judge -> normal exit 1, not an uncaught crash.
-    assert excinfo.value.code == 1
+    # Inconclusive judge (issue #299) -> its own exit code, never an uncaught
+    # crash and never sharing exit 1 with a genuine JUDGE MISMATCH.
+    assert excinfo.value.code == 3
     assert docker_calls == ["run"]  # the agent did run
 
     # The core regression: its results must not be silently discarded.
@@ -250,6 +251,38 @@ def explode(*args: Any, **kwargs: Any) -> dict:
     assert meta["pass"] is False
 
 
+def test_judge_mismatch_keeps_exit_code_1_distinct_from_inconclusive(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    """A judge that actually renders match=False is a real result, not an
+    outage: it must keep exit 1, not be folded into #299's exit 3."""
+    run_mod, written, docker_calls = _prepare_run(monkeypatch, tmp_path, "model-a")
+
+    import clawbench.runner.judge as judge_mod
+
+    monkeypatch.setattr(
+        judge_mod,
+        "judge_request",
+        lambda *a, **k: {
+            "match": False,
+            "reason": "did not fulfill the instruction",
+            "judge_model": "model-a",
+            "raw": "{}",
+            "error": None,
+        },
+    )
+
+    with pytest.raises(SystemExit) as excinfo:
+        run_mod.main()
+
+    assert excinfo.value.code == 1
+    assert docker_calls == ["run"]
+    meta = written[0][1]
+    assert meta["judge_match"] is False
+    assert meta["pass"] is False
+
+
 def test_model_config_error_is_catchable_as_a_plain_exception(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None: