diff --git a/CHANGELOG.md b/CHANGELOG.md index ec9a25f2..64fa4dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed +- The `claw-eval` port now uses the same `test-cases///task.json` layout as the native corpora, instead of flat `.json` files. Case discovery in `clawbench-batch` and the TUI is a plain `*/task.json` search again, and the `validate-task` workflow covers the suite without special-casing. ## [0.10.0] - 2026-08-30 ### Added diff --git a/src/clawbench/runner/batch.py b/src/clawbench/runner/batch.py index 8d3637e5..54d7f54d 100644 --- a/src/clawbench/runner/batch.py +++ b/src/clawbench/runner/batch.py @@ -81,7 +81,7 @@ def discover_models(patterns: list[str] | None, all_models: bool) -> list[str]: def _case_id(d: Path) -> int | None: - """Extract the numeric task ID from V1/V2/flat Claw-Eval case names.""" + """Extract the numeric task ID from V1/V2/Claw-Eval case names.""" match = re.match(r"^(?:v\d+-|ce-)?[A-Za-z]?(\d+)", d.stem) if not match: return None @@ -104,16 +104,8 @@ def _resolve_cases_dir(cases_dir: str | Path) -> Path: return path -def _flat_case_files(base: Path) -> list[Path]: - return [ - p - for p in base.glob("*.json") - if p.is_file() and p.name not in {"eligibility-report.json"} - ] - - def _all_cases_in(base: Path) -> list[Path]: - return [p.parent for p in base.glob("*/task.json")] + _flat_case_files(base) + return [p.parent for p in base.glob("*/task.json")] def discover_cases( @@ -139,8 +131,6 @@ def discover_cases( for d in expanded: if d.is_dir() and (d / "task.json").exists(): dirs.append(d) - elif d.is_file() and d.suffix == ".json": - dirs.append(d) elif case_range: dirs = sorted(_all_cases_in(base), key=_case_sort_key) else: diff --git a/src/clawbench/tui.py b/src/clawbench/tui.py index 9cdfbd99..93ac165c 100644 --- a/src/clawbench/tui.py +++ b/src/clawbench/tui.py @@ -383,13 +383,8 @@ def _case_sort_key(case: str) -> tuple[int, int, str]: def load_cases(cases_dir_name: str = "test-cases") -> list[str]: cases_dir = ASSET_ROOT / cases_dir_name - flat_cases = [ - p.stem - for p in cases_dir.glob("*.json") - if p.name not in {"eligibility-report.json"} - ] cases = sorted( - [*(p.parent.name for p in cases_dir.glob("*/task.json")), *flat_cases], + (p.parent.name for p in cases_dir.glob("*/task.json")), key=_case_sort_key, ) if not cases: diff --git a/test-cases/claw-eval/ce-T045zh-cve-research.json b/test-cases/claw-eval/ce-T045zh-cve-research/task.json similarity index 100% rename from test-cases/claw-eval/ce-T045zh-cve-research.json rename to test-cases/claw-eval/ce-T045zh-cve-research/task.json diff --git a/test-cases/claw-eval/ce-T046-cve-research.json b/test-cases/claw-eval/ce-T046-cve-research/task.json similarity index 100% rename from test-cases/claw-eval/ce-T046-cve-research.json rename to test-cases/claw-eval/ce-T046-cve-research/task.json diff --git a/test-cases/claw-eval/ce-T047zh-oss-comparison.json b/test-cases/claw-eval/ce-T047zh-oss-comparison/task.json similarity index 100% rename from test-cases/claw-eval/ce-T047zh-oss-comparison.json rename to test-cases/claw-eval/ce-T047zh-oss-comparison/task.json diff --git a/test-cases/claw-eval/ce-T048-oss-comparison.json b/test-cases/claw-eval/ce-T048-oss-comparison/task.json similarity index 100% rename from test-cases/claw-eval/ce-T048-oss-comparison.json rename to test-cases/claw-eval/ce-T048-oss-comparison/task.json diff --git a/test-cases/claw-eval/ce-T049zh-regulatory-research.json b/test-cases/claw-eval/ce-T049zh-regulatory-research/task.json similarity index 100% rename from test-cases/claw-eval/ce-T049zh-regulatory-research.json rename to test-cases/claw-eval/ce-T049zh-regulatory-research/task.json diff --git a/test-cases/claw-eval/ce-T050-regulatory-research.json b/test-cases/claw-eval/ce-T050-regulatory-research/task.json similarity index 100% rename from test-cases/claw-eval/ce-T050-regulatory-research.json rename to test-cases/claw-eval/ce-T050-regulatory-research/task.json diff --git a/test-cases/claw-eval/ce-T053-finance-us-steel-merger.json b/test-cases/claw-eval/ce-T053-finance-us-steel-merger/task.json similarity index 100% rename from test-cases/claw-eval/ce-T053-finance-us-steel-merger.json rename to test-cases/claw-eval/ce-T053-finance-us-steel-merger/task.json diff --git a/test-cases/claw-eval/ce-T054-finance-nflx-arppu-trend.json b/test-cases/claw-eval/ce-T054-finance-nflx-arppu-trend/task.json similarity index 100% rename from test-cases/claw-eval/ce-T054-finance-nflx-arppu-trend.json rename to test-cases/claw-eval/ce-T054-finance-nflx-arppu-trend/task.json diff --git a/test-cases/claw-eval/ce-T059-finance-abnb-cfo.json b/test-cases/claw-eval/ce-T059-finance-abnb-cfo/task.json similarity index 100% rename from test-cases/claw-eval/ce-T059-finance-abnb-cfo.json rename to test-cases/claw-eval/ce-T059-finance-abnb-cfo/task.json diff --git a/test-cases/claw-eval/ce-T060-finance-tko-endeavor-cost.json b/test-cases/claw-eval/ce-T060-finance-tko-endeavor-cost/task.json similarity index 100% rename from test-cases/claw-eval/ce-T060-finance-tko-endeavor-cost.json rename to test-cases/claw-eval/ce-T060-finance-tko-endeavor-cost/task.json diff --git a/test-cases/claw-eval/ce-T061-finance-mu-gm-beat.json b/test-cases/claw-eval/ce-T061-finance-mu-gm-beat/task.json similarity index 100% rename from test-cases/claw-eval/ce-T061-finance-mu-gm-beat.json rename to test-cases/claw-eval/ce-T061-finance-mu-gm-beat/task.json diff --git a/test-cases/claw-eval/ce-T062-finance-pltr-cagr.json b/test-cases/claw-eval/ce-T062-finance-pltr-cagr/task.json similarity index 100% rename from test-cases/claw-eval/ce-T062-finance-pltr-cagr.json rename to test-cases/claw-eval/ce-T062-finance-pltr-cagr/task.json diff --git a/test-cases/claw-eval/ce-T063-finance-fnd-sssg.json b/test-cases/claw-eval/ce-T063-finance-fnd-sssg/task.json similarity index 100% rename from test-cases/claw-eval/ce-T063-finance-fnd-sssg.json rename to test-cases/claw-eval/ce-T063-finance-fnd-sssg/task.json diff --git a/test-cases/claw-eval/ce-T064-finance-nflx-cash-req.json b/test-cases/claw-eval/ce-T064-finance-nflx-cash-req/task.json similarity index 100% rename from test-cases/claw-eval/ce-T064-finance-nflx-cash-req.json rename to test-cases/claw-eval/ce-T064-finance-nflx-cash-req/task.json diff --git a/test-cases/claw-eval/ce-T065-finance-x-inv-turnover.json b/test-cases/claw-eval/ce-T065-finance-x-inv-turnover/task.json similarity index 100% rename from test-cases/claw-eval/ce-T065-finance-x-inv-turnover.json rename to test-cases/claw-eval/ce-T065-finance-x-inv-turnover/task.json diff --git a/test-cases/claw-eval/ce-T066-finance-bros-gross-profit.json b/test-cases/claw-eval/ce-T066-finance-bros-gross-profit/task.json similarity index 100% rename from test-cases/claw-eval/ce-T066-finance-bros-gross-profit.json rename to test-cases/claw-eval/ce-T066-finance-bros-gross-profit/task.json diff --git a/test-cases/claw-eval/ce-T067zh-synopsys-china-revenue.json b/test-cases/claw-eval/ce-T067zh-synopsys-china-revenue/task.json similarity index 100% rename from test-cases/claw-eval/ce-T067zh-synopsys-china-revenue.json rename to test-cases/claw-eval/ce-T067zh-synopsys-china-revenue/task.json diff --git a/test-cases/claw-eval/ce-T069-micron-capex-analysis.json b/test-cases/claw-eval/ce-T069-micron-capex-analysis/task.json similarity index 100% rename from test-cases/claw-eval/ce-T069-micron-capex-analysis.json rename to test-cases/claw-eval/ce-T069-micron-capex-analysis/task.json diff --git a/test-cases/claw-eval/ce-T071-video-mme-coauthor-papers.json b/test-cases/claw-eval/ce-T071-video-mme-coauthor-papers/task.json similarity index 100% rename from test-cases/claw-eval/ce-T071-video-mme-coauthor-papers.json rename to test-cases/claw-eval/ce-T071-video-mme-coauthor-papers/task.json diff --git a/tests/test_host_tasks.py b/tests/test_host_tasks.py index 8adfc6b5..e3a2b992 100644 --- a/tests/test_host_tasks.py +++ b/tests/test_host_tasks.py @@ -18,13 +18,7 @@ def _suite_base(suite: str) -> Path: def _task_files_for_suite(suite: str) -> list[Path]: base = _suite_base(suite) - task_files = [path for path in base.glob("*/task.json") if path.is_file()] - task_files.extend( - path - for path in base.glob("*.json") - if path.is_file() and path.name != "eligibility-report.json" - ) - return sorted(task_files) + return sorted(path for path in base.glob("*/task.json") if path.is_file()) @pytest.mark.parametrize("suite", sorted(batch.CASE_SUITES)) @@ -37,10 +31,8 @@ def test_builtin_case_suites_are_discoverable(suite: str) -> None: assert cases, f"{suite} should contain at least one case" for case in cases: - if case.is_dir(): - assert (case / "task.json").is_file() - else: - assert case.is_file() + assert case.is_dir(), f"{case} should be a task directory" + assert (case / "task.json").is_file() @pytest.mark.parametrize("suite", sorted(batch.CASE_SUITES)) diff --git a/tests/test_tui_helpers.py b/tests/test_tui_helpers.py index 818d4280..49c500e9 100644 --- a/tests/test_tui_helpers.py +++ b/tests/test_tui_helpers.py @@ -129,7 +129,7 @@ def test_tui_dataset_and_case_helpers_use_known_suites() -> None: ) -def test_tui_load_cases_reads_flat_and_directory_suites() -> None: +def test_tui_load_cases_reads_every_builtin_suite() -> None: v1_cases = tui.load_cases("test-cases/v1") claw_eval_cases = tui.load_cases("test-cases/claw-eval")