diff --git a/CHANGELOG.md b/CHANGELOG.md index 64fa4dea..ef1b4197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed - The `claw-eval` port now uses the same `test-cases///task.json` layout as the native corpora, instead of flat `.json` files. Case discovery in `clawbench-batch` and the TUI is a plain `*/task.json` search again, and the `validate-task` workflow covers the suite without special-casing. +### Fixed +- Fixed a judge-provider outage (or an unparseable judge reply) being recorded as an agent failure. `run.py` now exits 3 instead of 1 when the judge never renders a verdict, `batch.py` gives it its own `judge_inconclusive` bucket in `batch-summary.json` instead of folding it into `failed`, and `clawbench-rescore` now retries a cached `match: null` verdict even without `--force`. + ## [0.10.0] - 2026-08-30 ### Added - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia). diff --git a/src/clawbench/eval/rescore.py b/src/clawbench/eval/rescore.py index 3df30a75..91b29e57 100644 --- a/src/clawbench/eval/rescore.py +++ b/src/clawbench/eval/rescore.py @@ -74,8 +74,11 @@ def rescore_one( judge_p = run_dir / JUDGE_FILE[rubric] if judge_p.exists() and not force: try: - out[rubric] = json.loads(judge_p.read_text()) - continue + cached = json.loads(judge_p.read_text()) + # A cached match=None is not a scored result: retry it. + if cached.get("match") is not None: + out[rubric] = cached + continue except Exception: pass verdict = judge_funcs[rubric](model_cfg, judge_model, instruction, intercept) @@ -280,7 +283,10 @@ def main() -> int: if not m.get("intercepted"): continue needs = any( - args.force or not (rd / JUDGE_FILE[r]).exists() for r in rubrics + args.force + or not (rd / JUDGE_FILE[r]).exists() + or json.loads((rd / JUDGE_FILE[r]).read_text()).get("match") is None + for r in rubrics ) if needs: pending.append(rd) diff --git a/src/clawbench/runner/batch.py b/src/clawbench/runner/batch.py index 54d7f54d..7b86f141 100644 --- a/src/clawbench/runner/batch.py +++ b/src/clawbench/runner/batch.py @@ -316,6 +316,11 @@ async def run_job( job.status = "passed" elif proc.returncode == 1: job.status = "failed" + elif proc.returncode == 3: + # run.py's own signal for "judge never rendered a verdict": + # kept out of "failed" so a judge outage cannot masquerade + # as the agent having failed the task. + job.status = "judge_inconclusive" else: job.status = "error" except asyncio.CancelledError: @@ -352,7 +357,7 @@ async def run_job( # Task cancelled while waiting on semaphore, throttle wait, or # before subprocess was created. "running" can appear here if # CancelledError hit after status was set but before proc started. - if job.status not in ("passed", "failed", "error"): + if job.status not in ("passed", "failed", "error", "judge_inconclusive"): job.status = "skipped" raise @@ -362,10 +367,12 @@ def print_progress(jobs: list[Job], start: float) -> None: running = sum(1 for j in jobs if j.status == "running") passed = sum(1 for j in jobs if j.status == "passed") failed = sum(1 for j in jobs if j.status in ("failed", "error")) + inconclusive = sum(1 for j in jobs if j.status == "judge_inconclusive") elapsed = fmt_duration(time.monotonic() - start) print( f"[{ts()}] [BATCH] {done}/{len(jobs)} done | {running} running | " - f"{passed} passed, {failed} failed | {elapsed} elapsed", + f"{passed} passed, {failed} failed, {inconclusive} judge-inconclusive | " + f"{elapsed} elapsed", file=sys.stderr, ) @@ -401,7 +408,7 @@ def print_summary( totals[j.status] = totals.get(j.status, 0) + 1 parts = [ f"{totals.get(s, 0)} {s}" - for s in ("passed", "failed", "error", "skipped") + for s in ("passed", "failed", "error", "judge_inconclusive", "skipped") if totals.get(s) ] print(f"\nTotal: {len(jobs)} jobs | {' | '.join(parts)}") @@ -566,7 +573,7 @@ def write_summary_json( ], "totals": { s: sum(1 for j in jobs if j.status == s) - for s in ("passed", "failed", "error", "skipped") + for s in ("passed", "failed", "error", "judge_inconclusive", "skipped") }, } (base_output / "batch-summary.json").write_text(json.dumps(data, indent=2)) diff --git a/src/clawbench/runner/run.py b/src/clawbench/runner/run.py index 4c9bf3d3..b6a43e17 100644 --- a/src/clawbench/runner/run.py +++ b/src/clawbench/runner/run.py @@ -794,7 +794,10 @@ def handle_sigint(sig, frame): f"\nINTERCEPTED but JUDGE {'MISMATCH' if verdict is False else 'INCONCLUSIVE'} " f"— results in {output_dir}\n reason: {reason[:200]}" ) - sys.exit(1) + # match=None means the judge never rendered a verdict (outage, retries + # exhausted, unparseable reply): give it its own exit code instead of + # sharing exit 1 with a genuine JUDGE MISMATCH. + sys.exit(1 if verdict is False else 3) if final_pass: status = "INTERCEPTED" if args.no_judge else "INTERCEPTED + JUDGE MATCH" print(f"\n{status} — results in {output_dir}") diff --git a/tests/test_batch_judge_inconclusive.py b/tests/test_batch_judge_inconclusive.py new file mode 100644 index 00000000..3af65f54 --- /dev/null +++ b/tests/test_batch_judge_inconclusive.py @@ -0,0 +1,99 @@ +"""Regression tests for issue #299: a judge outage must not be recorded as an +agent failure. + +run.py now exits 3, instead of sharing exit 1 with a genuine JUDGE MISMATCH, +when the judge never renders a verdict (match=None). These tests drive +batch.py's run_job() against a mocked subprocess to lock in that exit 3 lands +in its own `judge_inconclusive` bucket, kept out of `failed`/`error`, all the +way into batch-summary.json's totals. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path + +import pytest + +from clawbench.runner import batch +from clawbench.runner.batch import Job, StartupThrottle, run_job, write_summary_json + + +class _FakeProc: + def __init__(self, returncode: int) -> None: + self.returncode = returncode + self.pid = 99999 + + async def communicate(self): + return b"", None + + +def _run_one_job( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, returncode: int +) -> Job: + batch.shutdown_event = asyncio.Event() + batch.running_procs.clear() + + async def fake_create_subprocess_exec(*args, **kwargs): + return _FakeProc(returncode) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec) + + job = Job(model="model-a", case_dir=tmp_path / "case", case_name="case") + log_dir = tmp_path / "batch-logs" + log_dir.mkdir() + + asyncio.run( + run_job( + job, + asyncio.Semaphore(1), + StartupThrottle(0), + tmp_path, + log_dir, + [job], + 0.0, + no_upload=True, + ) + ) + return job + + +@pytest.mark.parametrize( + "returncode,expected_status", + [(0, "passed"), (1, "failed"), (3, "judge_inconclusive"), (2, "error")], +) +def test_run_job_maps_exit_code_to_status( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + returncode: int, + expected_status: str, +) -> None: + job = _run_one_job(monkeypatch, tmp_path, returncode) + assert job.status == expected_status + + +def test_judge_inconclusive_counted_separately_in_batch_summary_json( + tmp_path: Path, +) -> None: + jobs = [ + Job(model="m", case_dir=tmp_path, case_name="passed-case", status="passed"), + Job(model="m", case_dir=tmp_path, case_name="failed-case", status="failed"), + Job( + model="m", + case_dir=tmp_path, + case_name="inconclusive-case", + status="judge_inconclusive", + ), + ] + + write_summary_json(jobs, tmp_path, elapsed=1.0, max_concurrent=1, started_at="now") + + data = json.loads((tmp_path / "batch-summary.json").read_text()) + assert data["totals"] == { + "passed": 1, + "failed": 1, + "error": 0, + "judge_inconclusive": 1, + "skipped": 0, + } diff --git a/tests/test_rescore_retarget.py b/tests/test_rescore_retarget.py new file mode 100644 index 00000000..9f840680 --- /dev/null +++ b/tests/test_rescore_retarget.py @@ -0,0 +1,75 @@ +"""Regression test for issue #299, ask 3: clawbench-rescore must retarget a +run whose cached judge verdict is `match: null` (the judge never rendered a +verdict last time) even without --force, since that cached file is not a +scored result: it is the same outage this issue is about, just replayed +from disk instead of from a live judge call. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from clawbench.eval.rescore import rescore_one + + +def _make_run_dir(tmp_path: Path) -> Path: + run_dir = tmp_path / "model-a" / "run-1" + (run_dir / "data").mkdir(parents=True) + (run_dir / "run-meta.json").write_text( + json.dumps({"intercepted": True, "instruction": "do the task"}) + ) + (run_dir / "data" / "interception.json").write_text( + json.dumps({"request": {"url": "https://example.test"}}) + ) + return run_dir + + +def test_cached_inconclusive_verdict_is_retried_without_force(tmp_path: Path) -> None: + run_dir = _make_run_dir(tmp_path) + (run_dir / "judge.json").write_text( + json.dumps({"match": None, "reason": "judge_call_failed: timeout"}) + ) + + calls = [] + + def fake_judge(model_cfg, judge_model, instruction, intercept): + calls.append(1) + return {"match": True, "reason": "fulfills it"} + + out = rescore_one( + model_cfg={}, + judge_model="judge-a", + run_dir=run_dir, + force=False, + rubrics=["strict"], + judge_funcs={"strict": fake_judge}, + ) + + assert len(calls) == 1 # retried despite force=False + assert out["strict"]["match"] is True + + +def test_cached_scored_verdict_is_not_retried_without_force(tmp_path: Path) -> None: + run_dir = _make_run_dir(tmp_path) + (run_dir / "judge.json").write_text( + json.dumps({"match": False, "reason": "did not fulfill it"}) + ) + + calls = [] + + def fake_judge(model_cfg, judge_model, instruction, intercept): + calls.append(1) + return {"match": True, "reason": "should not be reached"} + + out = rescore_one( + model_cfg={}, + judge_model="judge-a", + run_dir=run_dir, + force=False, + rubrics=["strict"], + judge_funcs={"strict": fake_judge}, + ) + + assert calls == [] # cached scored verdict wins, no retry + assert out["strict"]["match"] is False diff --git a/tests/test_run_judge_stage.py b/tests/test_run_judge_stage.py index d911492f..9bad087e 100644 --- a/tests/test_run_judge_stage.py +++ b/tests/test_run_judge_stage.py @@ -237,8 +237,9 @@ def explode(*args: Any, **kwargs: Any) -> dict: with pytest.raises(SystemExit) as excinfo: run_mod.main() - # Inconclusive judge -> normal exit 1, not an uncaught crash. - assert excinfo.value.code == 1 + # Inconclusive judge (issue #299) -> its own exit code, never an uncaught + # crash and never sharing exit 1 with a genuine JUDGE MISMATCH. + assert excinfo.value.code == 3 assert docker_calls == ["run"] # the agent did run # The core regression: its results must not be silently discarded. @@ -250,6 +251,38 @@ def explode(*args: Any, **kwargs: Any) -> dict: assert meta["pass"] is False +def test_judge_mismatch_keeps_exit_code_1_distinct_from_inconclusive( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A judge that actually renders match=False is a real result, not an + outage: it must keep exit 1, not be folded into #299's exit 3.""" + run_mod, written, docker_calls = _prepare_run(monkeypatch, tmp_path, "model-a") + + import clawbench.runner.judge as judge_mod + + monkeypatch.setattr( + judge_mod, + "judge_request", + lambda *a, **k: { + "match": False, + "reason": "did not fulfill the instruction", + "judge_model": "model-a", + "raw": "{}", + "error": None, + }, + ) + + with pytest.raises(SystemExit) as excinfo: + run_mod.main() + + assert excinfo.value.code == 1 + assert docker_calls == ["run"] + meta = written[0][1] + assert meta["judge_match"] is False + assert meta["pass"] is False + + def test_model_config_error_is_catchable_as_a_plain_exception( monkeypatch: pytest.MonkeyPatch, ) -> None: