From e3bc0a054778bef740894e07c7ffb993575f5a84 Mon Sep 17 00:00:00 2001 From: vaibhavdabas16 Date: Wed, 2 Sep 2026 23:34:15 +0530 Subject: [PATCH] feat(batch): report Stage 1 and Stage 2 together, never one alone The end-of-batch stats printed a single headline number: 78/156 intercepted which is the number #243 says must never stand on its own. Raw Stage-1 interception overcounts success by about 2x -- the interceptor sees the right request, the judge is what decides whether it carried the right intent. On V2, 156 intercepted runs came back with 78 judge-confirmed: 50% precision. Quoting interception alone makes an agent look twice as good as it is; quoting the judged rate alone hides how much of the corpus the interceptor reached. batch-summary.json was worse off: its totals are job statuses, where "passed" already folds both stages into one verdict, so the file carried no Stage-1 number at all. Anything downstream had to re-walk every run directory to recover it. - Split the run-directory walk out of print_run_stats into collect_run_rows(), which now also reads judge_match and the judge model from run-meta.json. - Add stage_totals() and format_stage_totals(). The formatter refuses to render Stage 1 alone: with --no-judge it says stage 2 was not run rather than letting interception stand as the score. - Per-run table gains a Stage1/Stage2 column pair. - batch-summary.json gains a "stages" block; job-status totals unchanged. A judge call that failed after retries counts as stage2_unjudged, not as a stage-2 failure -- it needs re-judging, it is not a verdict. For the same reason stage2_rate is null rather than 0.0 when no judge ran. Scope note: the leaderboard rows and README tables #243 also asks about live outside this repo (the HF dataset and the website). This covers the artifacts this repo produces, and documents the rule in eval/scoring.md. --- CHANGELOG.md | 5 ++ docs/cli.md | 2 +- eval/scoring.md | 20 +++++ src/clawbench/runner/batch.py | 130 +++++++++++++++++++++++++---- tests/test_two_stage_reporting.py | 131 ++++++++++++++++++++++++++++++ 5 files changed, 273 insertions(+), 15 deletions(-) create mode 100644 tests/test_two_stage_reporting.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ec9a25f2..07dd5f7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added +- `batch-summary.json` now carries a `stages` block with the Stage-1 (interception) and Stage-2 (judged) counts, rates, and the judge model, so both stages can be reported without re-walking every run directory. + +### Changed +- `clawbench-batch` per-run stats now show a `Stage1`/`Stage2` column pair and end with a line reporting both stages plus stage-1 precision, instead of the interception count alone. ## [0.10.0] - 2026-08-30 ### Added diff --git a/docs/cli.md b/docs/cli.md index 9f3bb254..404d4c21 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -77,7 +77,7 @@ Execution: | `--dry-run` | off | Print the job matrix without running anything | | `--output-dir ` | `test-output` | Base output directory | -`--harness`, `--judge`, `--no-judge`, `--no-upload`, and the `--browser-*` flags behave as in `clawbench-run`. A `batch-summary.json` is written alongside the per-run directories. +`--harness`, `--judge`, `--no-judge`, `--no-upload`, and the `--browser-*` flags behave as in `clawbench-run`. A `batch-summary.json` is written alongside the per-run directories; its `stages` block carries the Stage-1 (interception) and Stage-2 (judged) counts separately, which is also what the end-of-batch console line prints. See [`eval/scoring.md`](../eval/scoring.md#always-report-both-stages) for why both are always reported together. ## `clawbench-rescore` diff --git a/eval/scoring.md b/eval/scoring.md index d1a2547b..358abdb8 100644 --- a/eval/scoring.md +++ b/eval/scoring.md @@ -121,6 +121,26 @@ Per-run record in `run-meta.json` gets: Empirically, requiring both moves headline scores down sharply (typical Stage-1-only is 1.5–2× Stage-2 numbers), surfacing models that "almost get there" vs. models that actually complete the task. The two-stage system also makes failure diagnosis cheap — the run-meta tells you which stage cut off. +## Always report both stages + +Every published number carries **both** stages, side by side, with the judge model named. Neither stands alone: + +- **Stage 1 alone overcounts success by roughly 2x.** In V2, `gemini-3.1-pro-preview` intercepted 69 tasks and the judge confirmed 37 (54% precision); `gemini-3.5-flash` intercepted 66 and confirmed 33 (50%). Aggregate: 156 intercepted, 78 confirmed — **50%**. Quoting interception on its own makes an agent look about twice as good as it is, because "right request, wrong intent" reads as a pass. +- **Stage 2 alone hides interception coverage.** Without the Stage-1 number there is no way to see how much of the corpus the interceptor reached, so a low judged rate is ambiguous between "the agent never got there" and "the agent got there with the wrong payload". + +The tooling reports both by construction: + +| Artifact | Stage 1 | Stage 2 | +|---|---|---| +| `batch-summary.json` | `stages.stage1_intercepted`, `stages.stage1_rate` | `stages.stage2_judged_match`, `stages.stage2_rate`, `stages.judge_models` | +| `clawbench-batch` console output | `stage 1 (intercepted): n/N` | `stage 2 (judged, ): n/N` plus stage-1 precision | +| `rescore-summary.json` | `n_intercepted`, `pass_rate_stage1_only` | `n_judge_match`, `pass_rate_with_judge` | +| `clawbench-analyze` report | Stage-1 rate | Stage-2 rate | + +`stages.stage1_precision` is Stage 2 over Stage 1 — the share of intercepted runs that survived the judge, i.e. the size of the "right request, wrong intent" gap for that batch. + +With `--no-judge` there is no Stage-2 number to report, and the output says so explicitly rather than letting the interception count stand as the score. A run whose judge call failed after retries is counted in `stages.stage2_unjudged`, not as a Stage-2 failure — it needs re-judging. + ## Aggregating to a leaderboard row Each (model × harness × corpus) batch produces one `rescore-summary.json`: diff --git a/src/clawbench/runner/batch.py b/src/clawbench/runner/batch.py index 8d3637e5..4f77fd27 100644 --- a/src/clawbench/runner/batch.py +++ b/src/clawbench/runner/batch.py @@ -14,6 +14,7 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path +from typing import Any import yaml @@ -431,13 +432,17 @@ def print_summary( print(f" ... and {len(bad) - 10} more") -def print_run_stats(base_output: Path) -> None: - """Print per-run statistics from output directories.""" - print(f"\n{'=' * 80}") - print("PER-RUN STATS") - print(f"{'=' * 80}") +def collect_run_rows(base_output: Path) -> list[dict[str, Any]]: + """Read per-run stats out of a batch output directory. - rows = [] + Both scoring stages are read here so no caller has to reach for one + without the other: ``intercepted`` is Stage 1 (the interceptor matched the + task's request schema) and ``judged`` is Stage 2 (the LLM judge confirmed + the payload fulfils the instruction). ``judged`` is None when no verdict + exists -- either the judge was not run (``--no-judge``) or it failed to + return one. + """ + rows: list[dict[str, Any]] = [] for model_dir in sorted(base_output.iterdir()): if not model_dir.is_dir() or model_dir.name.startswith("batch-"): continue @@ -462,12 +467,22 @@ def print_run_stats(base_output: Path) -> None: and browser_runtime.get("recording_mode") == "provider" and browser_runtime.get("recording_url") ) + judged = meta.get("judge_match") + judged = judged if isinstance(judged, bool) else None + judge_attempted = "judge_match" in meta + run_flags = meta.get("run_flags") + judge_model = ( + run_flags.get("judge") if isinstance(run_flags, dict) else None + ) else: case = run_dir.name model = model_dir.name intercepted = False duration = 0 provider_recording = False + judged = None + judge_attempted = False + judge_model = None # Count actions actions_file = data / "actions.jsonl" @@ -495,9 +510,89 @@ def print_run_stats(base_output: Path) -> None: "provider_recording": provider_recording, "duration": duration, "intercepted": intercepted, + "judged": judged, + "judge_attempted": judge_attempted, + "judge_model": judge_model, } ) + return rows + + +def stage_totals(rows: list[dict[str, Any]]) -> dict[str, Any]: + """Count both scoring stages together. + + Stage-1 interception on its own overcounts success by roughly 2x -- the + interceptor sees the right request; the judge is what decides whether it + carried the right intent (#243). Reporting only Stage 1 makes an agent look + about twice as good as it is, and reporting only Stage 2 hides how much of + the corpus the interceptor covered. So this returns both, plus the + precision between them, and every caller prints all of it. + """ + runs = len(rows) + intercepted = sum(1 for r in rows if r["intercepted"]) + judged_match = sum(1 for r in rows if r["judged"] is True) + unjudged = sum( + 1 + for r in rows + if r["intercepted"] and r["judge_attempted"] and r["judged"] is None + ) + judge_ran = any(r["judge_attempted"] for r in rows) + return { + "runs": runs, + "stage1_intercepted": intercepted, + "stage1_rate": round(intercepted / runs, 4) if runs else None, + "stage2_judged_match": judged_match, + # None, not 0.0: with --no-judge there is no stage-2 rate to quote, and + # a zero would read as "the judge rejected everything". + "stage2_rate": round(judged_match / runs, 4) if runs and judge_ran else None, + "stage2_unjudged": unjudged, + "stage1_precision": ( + round(judged_match / intercepted, 4) if intercepted and judge_ran else None + ), + "judge_models": sorted({r["judge_model"] for r in rows if r["judge_model"]}), + "judge_ran": judge_ran, + } + + +def format_stage_totals(totals: dict[str, Any]) -> str: + """One line carrying both stages, never one of them. + + The point of #243 is that a single headline number gets misread, so this + refuses to render Stage 1 alone: when the judge did not run it says so + explicitly rather than letting the interception count stand as "the" score. + """ + runs = totals["runs"] + stage1 = f"stage 1 (intercepted): {totals['stage1_intercepted']}/{runs}" + if totals["stage1_rate"] is not None: + stage1 += f" ({totals['stage1_rate']:.0%})" + if not totals["judge_ran"]: + return ( + f"{stage1} | stage 2 (judged): not run" + " -- stage 1 alone overcounts success" + ) + + judges = ", ".join(totals["judge_models"]) or "unknown judge" + stage2 = f"stage 2 (judged, {judges}): {totals['stage2_judged_match']}/{runs}" + if totals["stage2_rate"] is not None: + stage2 += f" ({totals['stage2_rate']:.0%})" + line = f"{stage1} | {stage2}" + if totals["stage1_precision"] is not None: + line += f" | stage-1 precision: {totals['stage1_precision']:.0%}" + if totals["stage2_unjudged"]: + line += f" | {totals['stage2_unjudged']} awaiting a verdict" + return line + + +def print_run_stats(base_output: Path) -> None: + """Print per-run statistics from output directories.""" + print("") + print("=" * 80) + print("PER-RUN STATS") + print("=" * 80) + + rows = collect_run_rows(base_output) + if not rows: print(" No run data found.") return @@ -507,11 +602,14 @@ def print_run_stats(base_output: Path) -> None: case_w = min(max(len(r["case"]) for r in rows), 50) model_w = max(len(r["model"]) for r in rows) - header = f"{'Case':<{case_w}} {'Model':<{model_w}} Actions Screenshots Recording Duration Intercepted" + header = f"{'Case':<{case_w}} {'Model':<{model_w}} Actions Screenshots Recording Duration Stage1 Stage2" print(header) print("-" * len(header)) for r in rows: - result = "yes" if r["intercepted"] else "no" + stage1 = "yes" if r["intercepted"] else "no" + # "-" is not a fail: it means no verdict exists for this run, either + # because --no-judge skipped stage 2 or because the judge returned none. + stage2 = "-" if r["judged"] is None else ("yes" if r["judged"] else "no") case = r["case"][:case_w] # Flag abnormal runs: no actions, no screenshots, no recording, or very short duration abnormal = ( @@ -527,14 +625,13 @@ def print_run_stats(base_output: Path) -> None: f"{case:<{case_w}} {r['model']:<{model_w}} " f"{r['actions']:>7} {r['screenshots']:>11} " f"{recording:>10} " - f"{fmt_duration(r['duration']):>8} {result}" + f"{fmt_duration(r['duration']):>8} {stage1:<6} {stage2}" ) if abnormal: print(f"{RED}{line}{RESET}") else: print(line) - total_pass = sum(1 for r in rows if r["intercepted"]) abnormal_count = sum( 1 for r in rows @@ -543,11 +640,10 @@ def print_run_stats(base_output: Path) -> None: or (not r["provider_recording"] and r["recording_mb"] < 0.5) or r["duration"] < 30 ) - print(f"\n{total_pass}/{len(rows)} intercepted", end="") + print("") + print(format_stage_totals(stage_totals(rows))) if abnormal_count: - print(f" | {RED}{abnormal_count} abnormal{RESET}") - else: - print() + print(f"{RED}{abnormal_count} abnormal{RESET}") def write_summary_json( @@ -559,6 +655,11 @@ def write_summary_json( browser_runtime: str = "local", ) -> None: now = datetime.now(timezone.utc).isoformat() + # Job status already folds both stages into one verdict ("passed" means + # intercepted AND judged). Carry the stages separately as well so a + # consumer of this file can report interception and judged rates without + # re-walking every run directory -- and so neither can be quoted alone. + stages = stage_totals(collect_run_rows(base_output)) data = { "started_at": started_at, "finished_at": now, @@ -574,6 +675,7 @@ def write_summary_json( } for j in jobs ], + "stages": stages, "totals": { s: sum(1 for j in jobs if j.status == s) for s in ("passed", "failed", "error", "skipped") diff --git a/tests/test_two_stage_reporting.py b/tests/test_two_stage_reporting.py new file mode 100644 index 00000000..b89952c2 --- /dev/null +++ b/tests/test_two_stage_reporting.py @@ -0,0 +1,131 @@ +"""Both scoring stages are reported together (#243).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from clawbench.runner import batch + + +def _write_run( + base: Path, + model: str, + case: str, + *, + intercepted: bool, + judge_match: bool | None | str = "absent", + judge_model: str | None = "deepseek-v4-pro", +) -> None: + run_dir = base / model / case + (run_dir / "data").mkdir(parents=True) + meta: dict = { + "test_case": case, + "model": model, + "intercepted": intercepted, + "duration_seconds": 300, + "run_flags": {"judge": judge_model}, + } + if judge_match != "absent": + meta["judge_match"] = judge_match + (run_dir / "run-meta.json").write_text(json.dumps(meta)) + + +def _batch_with_known_stage_split(tmp_path: Path) -> Path: + """Four runs: 3 intercepted, 1 of those judged a match, 1 unjudged.""" + base = tmp_path / "test-output" + _write_run(base, "model-a", "case-1", intercepted=True, judge_match=True) + _write_run(base, "model-a", "case-2", intercepted=True, judge_match=False) + _write_run(base, "model-a", "case-3", intercepted=True, judge_match=None) + _write_run(base, "model-a", "case-4", intercepted=False, judge_match=False) + return base + + +def test_stage_totals_separates_interception_from_judged(tmp_path: Path) -> None: + rows = batch.collect_run_rows(_batch_with_known_stage_split(tmp_path)) + totals = batch.stage_totals(rows) + + assert totals["runs"] == 4 + assert totals["stage1_intercepted"] == 3 + assert totals["stage1_rate"] == 0.75 + assert totals["stage2_judged_match"] == 1 + assert totals["stage2_rate"] == 0.25 + # 1 of 3 intercepted runs survived the judge: the "right request, wrong + # intent" gap this issue is about. + assert totals["stage1_precision"] == round(1 / 3, 4) + assert totals["judge_models"] == ["deepseek-v4-pro"] + assert totals["judge_ran"] is True + + +def test_a_failed_judge_call_is_not_a_stage_2_failure(tmp_path: Path) -> None: + """judge_match=None means no verdict, which is not the same as 'no match'.""" + rows = batch.collect_run_rows(_batch_with_known_stage_split(tmp_path)) + totals = batch.stage_totals(rows) + + assert totals["stage2_unjudged"] == 1 + assert totals["stage2_judged_match"] == 1 + + +def test_no_judge_run_reports_the_absence_instead_of_a_stage_1_headline( + tmp_path: Path, +) -> None: + base = tmp_path / "test-output" + _write_run(base, "model-a", "case-1", intercepted=True, judge_model=None) + _write_run(base, "model-a", "case-2", intercepted=False, judge_model=None) + + totals = batch.stage_totals(batch.collect_run_rows(base)) + line = batch.format_stage_totals(totals) + + assert totals["judge_ran"] is False + assert totals["stage2_rate"] is None + assert "stage 1 (intercepted): 1/2" in line + assert "not run" in line + # The interception count must never be the only number on the line. + assert "stage 2" in line + + +def test_summary_line_carries_both_stages_and_the_judge_model( + tmp_path: Path, +) -> None: + totals = batch.stage_totals( + batch.collect_run_rows(_batch_with_known_stage_split(tmp_path)) + ) + line = batch.format_stage_totals(totals) + + assert "stage 1 (intercepted): 3/4 (75%)" in line + assert "stage 2 (judged, deepseek-v4-pro): 1/4 (25%)" in line + assert "stage-1 precision: 33%" in line + assert "1 awaiting a verdict" in line + + +def test_batch_summary_json_carries_both_stages(tmp_path: Path) -> None: + base = _batch_with_known_stage_split(tmp_path) + jobs = [ + batch.Job(case_dir=Path("case-1"), case_name="case-1", model="model-a"), + ] + jobs[0].status = "passed" + + batch.write_summary_json( + jobs, + base, + elapsed=12.0, + max_concurrent=2, + started_at="2026-01-01T00:00:00+00:00", + ) + + summary = json.loads((base / "batch-summary.json").read_text()) + assert summary["stages"]["stage1_intercepted"] == 3 + assert summary["stages"]["stage2_judged_match"] == 1 + assert summary["stages"]["judge_models"] == ["deepseek-v4-pro"] + # The pre-existing job-status totals are unchanged. + assert summary["totals"]["passed"] == 1 + + +def test_stage_totals_on_an_empty_batch_does_not_divide_by_zero() -> None: + totals = batch.stage_totals([]) + + assert totals["runs"] == 0 + assert totals["judge_ran"] is False + assert totals["stage1_rate"] is None + assert totals["stage2_rate"] is None + assert totals["stage1_precision"] is None