diff --git a/.github/workflows/mcb-public-benchmarks.yml b/.github/workflows/mcb-public-benchmarks.yml new file mode 100644 index 00000000..c49ada2c --- /dev/null +++ b/.github/workflows/mcb-public-benchmarks.yml @@ -0,0 +1,58 @@ +name: MCB public benchmark integration + +on: + pull_request: + paths: + - "experiments/model-capability-benchmark/**" + - "packages/benchmark-core/**" + - ".github/workflows/mcb-public-benchmarks.yml" + workflow_dispatch: + +jobs: + public-benchmarks: + runs-on: ubuntu-latest + timeout-minutes: 25 + defaults: + run: + working-directory: experiments/model-capability-benchmark + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install uv + run: python -m pip install --disable-pip-version-check uv + - name: Sync locked dependencies + run: uv sync --frozen --extra dev + - name: Lint + run: uv run --frozen ruff check src tests/test_public_benchmarks.py + - name: Test public adapters and evaluators + run: uv run --frozen pytest -q tests/test_public_benchmarks.py + - name: Plan public Core without provider calls + run: uv run --frozen model-bench --suite suite_public.yaml plan --profile core --capabilities all + - name: Smoke-test real pinned upstream data without provider calls + run: | + uv run --frozen python - <<'PY' + from pathlib import Path + from benchmark_core import DatasetLoadContext + from model_capability_bench import load_capability_suite + suite = load_capability_suite(Path.cwd(), suite_filename="suite_public.yaml") + cache = Path.cwd() / ".cache/public-ci" + expected = { + "gsm8k-main-test": 12, + "squad-v2-validation": 12, + "jsonschemabench-github-easy-test": 12, + } + for dataset_id, count in expected.items(): + result = suite.datasets.load( + dataset_id, + DatasetLoadContext( + cache_dir=cache, + profile=suite.profiles["smoke"], + seed=42, + ), + ) + assert len(result.samples) == count, (dataset_id, len(result.samples)) + assert result.source_checksums.get("source") + print(dataset_id, len(result.samples), result.available_count, result.selection_fingerprint) + PY diff --git a/experiments/model-capability-benchmark/PUBLIC_BENCHMARKS.md b/experiments/model-capability-benchmark/PUBLIC_BENCHMARKS.md new file mode 100644 index 00000000..be19fa1b --- /dev/null +++ b/experiments/model-capability-benchmark/PUBLIC_BENCHMARKS.md @@ -0,0 +1,77 @@ +# MCB public benchmark suite (opt-in) + +MCB has two independent suites. **The historical default stays unchanged**: + +- `suite.yaml` / `capability-core` v3: 468 controlled/public-label cases, including all previous benchmark revisions. +- `suite_public.yaml` / `capability-public-benchmarks` v1: 140 small, reproducible public benchmark test/validation cases in Core. + +Public suite is an **MCB protocol on pinned public data**, **not** the official full dataset leaderboard protocol. +We do not publish a 40-case GSM8K score as an official GSM8K full-test score. + +## Sources and provenance + +| MCB dataset ID | Upstream | Upstream split | License | Core | +| --- | --- | --- | --- | ---: | +| `gsm8k-main-test` | [openai/gsm8k](https://huggingface.co/datasets/openai/gsm8k) | `main/test` (1,319 total) | MIT | 40 | +| `squad-v2-validation` | [rajpurkar/squad_v2](https://huggingface.co/datasets/rajpurkar/squad_v2) | `validation` (11,873 total) | CC BY-SA 4.0 | 60 (30 answerable + 30 unanswerable) | +| `jsonschemabench-github-easy-test` | [epfl-dlab/JSONSchemaBench](https://huggingface.co/datasets/epfl-dlab/JSONSchemaBench) | `Github_easy/test` (577 upstream) | MIT | 40 | + +Each source URL in `datasets.yaml` pins a **full immutable upstream commit SHA**, the adapter uses the original Parquet file, and the cached source's SHA-256 is preserved in `source_checksums`. Dataset IDs, revisions, selected source indices, source splits, profiles and selection fingerprints persist in evidence. Case IDs never depend on random order. This requires no new dependency: MCB reads Parquet with its existing DuckDB dependency. + +The upstream dataset may have known contamination in pretrained models. Public provenance alone does **not** prove model training independence. + +**SQuAD protocol:** MCB asks for JSON `{answer, abstain}`, uses the official SQuAD text normalization (lowercase, remove punctuation, articles and extra whitespace), and accepts every upstream gold alias. It scores `qa_correct`, which requires a correct answer on answerable items and a correct abstention on impossible items. It does not compute the official SQuAD v2 threshold-tuned F1/EM leaderboard metric. + +**GSM8K protocol:** MCB extracts the upstream final numeric gold after `####`, normalizes decimals/commas and evaluates the final answer. The original worked solution is not leaked to the prompt. This is an MCB zero-shot JSON-output evaluation, not necessarily prompt-parity with an external published number. + +**JSONSchemaBench protocol:** MCB tests JSON Schema compliance, **not** semantic extraction or JSONSchemaBench's official constrained-decoding engine timings/coverage. Its v1 adapter deliberately selects only object-root schemas that satisfy local validator checks, have locally resolvable references and at most 12,000 schema characters. Rejected upstream rows are counted (`source_rows`, `excluded_rows`, `eligible_rows`) to make this reduced subset explicit. Unsupported remote references are never automatically fetched. The upstream dataset has no gold data values: `expected=None` is correct. + +## Commands + +From `experiments/model-capability-benchmark`: + +```bash +uv sync --frozen --extra dev + +# Read configuration before downloading anything +uv run --frozen model-bench --suite suite_public.yaml capabilities +uv run --frozen model-bench --suite suite_public.yaml plan --profile core --capabilities all + +# Real preflight and cost/latency pilot on one selected model +export MODEL=qwen3.5-2b-q4km +uv run --frozen model-bench --suite suite_public.yaml validate-config --models "$MODEL" +uv run --frozen model-bench --suite suite_public.yaml estimate \ + --models "$MODEL" --profile core --capabilities all --pilot-cases 3 + +# Distinct run group: never overwrite legacy results +uv run --frozen model-bench --suite suite_public.yaml run \ + --run-group public-benchmarks-v1 \ + --models "$MODEL" --capabilities all --profile core + +# Offline projection and dashboard export from immutable run evidence +uv run --frozen model-bench project --rebuild --export-dashboard +``` + +Existing default commands without `--suite` still run historical `suite.yaml`. Use the same +public suite filename, seed, profile, commit, source artifact and evaluator versions for every compared model. + +Case budgets: `smoke = 12+12+12 = 36`, `core = 40+60+40 = 140`, +`extended = 200+200+150 = 550`. Source files are cached and downloaded only on first +load. Estimated run time is model-dependent. `plan` does not contact models or download data; +`estimate` and `run` do. + +## Science and interpretation guards + +- MCB stores **raw sources and actual selected case fingerprints**, so any same-case paired comparison remains reproducible. +- Do not compare public v1 scores directly to controlled v2/v3 scores, or call them equivalent despite both measuring math or QA. +- Do not give official benchmark leaderboard labels or cite official test totals as run sample counts. +- Report paired `n`, confidence intervals, invalid-output rates and missing resource data separately. +- SQuAD and JSONSchemaBench use different concepts of correctness; aggregating into one undifferentiated score is not recommended. +- JSONSchemaBench's object/root-size/reference filtering must remain versioned and visible, because it affects measured difficulty. +- Before external publication, perform a real-provider smoke + canonical run and review a sample of gold normalization and schema evaluator outputs. + +## Recommended follow-on + +Add a second JSONSchemaBench subset (`Github_medium`) only after the object-root compliance path passes +real provider validation. Add structured-output **semantic extraction** from benchmark sources separately; +schema compliance alone cannot replace the original controlled extraction suite. diff --git a/experiments/model-capability-benchmark/README.md b/experiments/model-capability-benchmark/README.md index e90a2e43..8441ecc4 100644 --- a/experiments/model-capability-benchmark/README.md +++ b/experiments/model-capability-benchmark/README.md @@ -103,6 +103,22 @@ read from the environment and is never written to the registry or manifest. See `.env.example` for the currently supported environment variables. +## Public benchmark suite (opt-in) + +A second, independently versioned suite (`suite_public.yaml`) integrates pinned upstream +test/validation subsets of **GSM8K**, **SQuAD 2.0** and **JSONSchemaBench**. +It has 36 smoke / 140 core / 550 extended cases and does not replace the canonical +468-case `suite.yaml`. The UI and persisted evidence keep the benchmark lineages distinct. +JSONSchemaBench measures schema compliance, not semantic field extraction. + +See **[PUBLIC_BENCHMARKS.md](PUBLIC_BENCHMARKS.md)** for sources, data licenses, +normalization/scoring semantics, reproducible run commands and the explicit +differences from official full-benchmark leaderboard protocols. + +```bash +uv run --frozen model-bench --suite suite_public.yaml plan --profile core --capabilities all +``` + ## Capability suite `suite.yaml` now composes five capability families: BANKING77 intent classification, diff --git a/experiments/model-capability-benchmark/datasets.yaml b/experiments/model-capability-benchmark/datasets.yaml index 277a8650..8f2764c0 100644 --- a/experiments/model-capability-benchmark/datasets.yaml +++ b/experiments/model-capability-benchmark/datasets.yaml @@ -156,3 +156,54 @@ datasets: path: data/math-reasoning-controlled-v3.yaml human_reviewed: false difficulty_labels: authored-provisional + + # Public benchmarks: immutable Hugging Face dataset revisions and original test splits. + gsm8k-main-test: + version: "1" + adapter: public-parquet + source: openai/gsm8k + revision: 740312add88f781978c0658806c59bc2815b9866 + split: test + license: MIT + sample_schema_version: "1" + cache_mode: download + options: + timeout_seconds: 120 + user_agent: model-capability-bench/0.1 + file: + url: https://huggingface.co/datasets/openai/gsm8k/resolve/740312add88f781978c0658806c59bc2815b9866/main/test-00000-of-00001.parquet + cache_path: gsm8k/740312add88f781978c0658806c59bc2815b9866/test.parquet + + squad-v2-validation: + version: "1" + adapter: public-parquet + source: rajpurkar/squad_v2 + revision: 3ffb306f725f7d2ce8394bc1873b24868140c412 + split: validation + license: CC-BY-SA-4.0 + sample_schema_version: "1" + cache_mode: download + options: + timeout_seconds: 120 + user_agent: model-capability-bench/0.1 + file: + url: https://huggingface.co/datasets/rajpurkar/squad_v2/resolve/3ffb306f725f7d2ce8394bc1873b24868140c412/squad_v2/validation-00000-of-00001.parquet + cache_path: squad-v2/3ffb306f725f7d2ce8394bc1873b24868140c412/validation.parquet + + jsonschemabench-github-easy-test: + version: "1" + adapter: public-parquet + source: epfl-dlab/JSONSchemaBench + revision: 83df923c5b1dd05fb5c9659ad751d7c056a3cd43 + split: test + license: MIT + sample_schema_version: "1" + cache_mode: download + options: + subset: Github_easy + max_schema_chars: 12000 + timeout_seconds: 120 + user_agent: model-capability-bench/0.1 + file: + url: https://huggingface.co/datasets/epfl-dlab/JSONSchemaBench/resolve/83df923c5b1dd05fb5c9659ad751d7c056a3cd43/Github_easy/test-00000-of-00001.parquet + cache_path: jsonschemabench/83df923c5b1dd05fb5c9659ad751d7c056a3cd43/github_easy_test.parquet diff --git a/experiments/model-capability-benchmark/profiles.yaml b/experiments/model-capability-benchmark/profiles.yaml index 5eae88ce..e8c2f94e 100644 --- a/experiments/model-capability-benchmark/profiles.yaml +++ b/experiments/model-capability-benchmark/profiles.yaml @@ -3,6 +3,9 @@ profiles: max_cases: 20 datasets: banking77: 77 + gsm8k-main-test: 12 + squad-v2-validation: 12 + jsonschemabench-github-easy-test: 12 clinc150-oos: 50 structured-output-controlled-v1: null qa-abstention-controlled-v1: null @@ -16,6 +19,9 @@ profiles: max_cases: 100 datasets: banking77: 154 + gsm8k-main-test: 40 + squad-v2-validation: 60 + jsonschemabench-github-easy-test: 40 clinc150-oos: 100 structured-output-controlled-v2: 60 qa-abstention-controlled-v2: 60 @@ -26,6 +32,9 @@ profiles: max_cases: 500 datasets: banking77: 770 + gsm8k-main-test: 200 + squad-v2-validation: 200 + jsonschemabench-github-easy-test: 150 clinc150-oos: 500 structured-output-controlled-v2: 60 qa-abstention-controlled-v2: 60 diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/cli.py b/experiments/model-capability-benchmark/src/model_capability_bench/cli.py index b27eb82b..9325fa6f 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/cli.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/cli.py @@ -60,7 +60,9 @@ def _selection(value: str, available: list[str]) -> tuple[str, ...]: def _select_capabilities(bundle, raw_value: str) -> tuple[str, ...]: canonical = [c.spec.capability_id for c in bundle.resolved_capabilities] - available = canonical + list(CAPABILITY_ALIASES.keys()) + available = canonical + [ + alias for alias, target in CAPABILITY_ALIASES.items() if target in canonical + ] selected = _selection(raw_value, available) return tuple(dict.fromkeys(resolve_capability_id(c) for c in selected)) @@ -71,6 +73,10 @@ def build_parser() -> argparse.ArgumentParser: description="Unified model capability benchmark runner.", ) parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument( + "--suite", default="suite.yaml", + help="Versioned suite YAML basename (e.g. suite_public.yaml); place before command.", + ) sub = parser.add_subparsers(dest="command", required=True) run = sub.add_parser("run", help="Execute a model × capability benchmark matrix.") @@ -284,7 +290,7 @@ def main() -> int: output = args.output_dir or root / "results" / "runs" / args.run_id summary = consolidate_runs( sources=args.run_dir, output_dir=output, run_id=args.run_id, - run_group=args.run_group, suite=load_capability_suite(root), plan_only=args.plan_only, + run_group=args.run_group, suite=load_capability_suite(root, suite_filename=args.suite), plan_only=args.plan_only, ) if not args.plan_only: reporting_config = load_reporting_config(root) @@ -448,7 +454,7 @@ def main() -> int: ) return 0 - bundle = load_capability_suite(root) + bundle = load_capability_suite(root, suite_filename=args.suite) defaults = load_runner_defaults(root) if args.command in {"models", "tasks", "datasets", "capabilities"}: diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py index aacae634..05639fb5 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py @@ -1,6 +1,7 @@ from model_capability_bench.datasets.banking77 import Banking77Dataset from model_capability_bench.datasets.clinc150 import Clinc150OosDataset from model_capability_bench.datasets.controlled_yaml import ControlledYamlDataset +from model_capability_bench.datasets.public_benchmarks import PublicParquetBenchmark from model_capability_bench.datasets.registry import ( build_dataset_plugins, build_dataset_registry, @@ -13,6 +14,7 @@ "Banking77Dataset", "Clinc150OosDataset", "ControlledYamlDataset", + "PublicParquetBenchmark", "StructuredOutputControlledDataset", "build_dataset_plugins", "build_dataset_registry", diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py new file mode 100644 index 00000000..81544c40 --- /dev/null +++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py @@ -0,0 +1,206 @@ +"""Pinned, offline-cacheable public benchmark Parquet adapters. + +The original test/validation rows and gold answers are not modified. MCB selects a +small deterministic subset, and records the exact upstream artifact checksum. +These are MCB subsets, not claims of official full-benchmark scores. +""" +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from decimal import Decimal, InvalidOperation + +from benchmark_core import ( + DatasetLoadContext, + DatasetLoadResult, + DatasetSpec, + Sample, + fingerprint_values, + sha256_file, +) +from jsonschema.exceptions import SchemaError +from jsonschema.validators import validator_for + +from model_capability_bench.datasets.common import single_cached_source +from model_capability_bench.datasets.selection import select_samples + + +def _rows(path, columns: tuple[str, ...]) -> list[dict]: + import duckdb + + selected = ", ".join(f'"{column}"' for column in columns) + with duckdb.connect(":memory:") as conn: + result = conn.execute( + f"SELECT {selected} FROM read_parquet(?)", [str(path)] + ) + names = [item[0] for item in result.description] + return [dict(zip(names, row, strict=True)) for row in result.fetchall()] + + +def _required_text(value: object, *, case: str, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{case}: missing or empty {field}") + return value + + +def _gsm_final_answer(solution: str) -> str: + match = re.search(r"(?m)^\s*####\s*([^\r\n]+)\s*$", solution) + if not match: + raise ValueError("GSM8K gold has no #### final answer") + text = match.group(1).strip().replace(",", "") + try: + result = Decimal(text) + except InvalidOperation as exc: + raise ValueError(f"Non-numeric GSM8K final answer: {text!r}") from exc + if not result.is_finite(): + raise ValueError("Non-finite GSM8K final answer") + return format(result.normalize(), "f") + + +def _squad_answers(value: object) -> tuple[str, ...]: + if not isinstance(value, Mapping): + raise TypeError("SQuAD answers must be an object") + texts = value.get("text") + if not isinstance(texts, list) or not all(isinstance(t, str) for t in texts): + raise ValueError("SQuAD answers.text must be a string list") + return tuple(dict.fromkeys(texts)) + + +def _unsafe_ref(value: object) -> bool: + if isinstance(value, Mapping): + for key, item in value.items(): + # Never fetch remote references implicitly while scoring. + if key in {"$ref", "$dynamicRef", "$recursiveRef"} and ( + not isinstance(item, str) or not item.startswith("#/") + ): + return True + if _unsafe_ref(item): + return True + elif isinstance(value, list): + return any(_unsafe_ref(item) for item in value) + return False + + +def _schema(value: object, *, max_chars: int) -> dict | None: + if not isinstance(value, str) or len(value) > max_chars: + return None + try: + schema = json.loads(value) + if not isinstance(schema, dict) or schema.get("type") != "object": + return None + if _unsafe_ref(schema): + return None + validator_for(schema).check_schema(schema) + return schema + except (ValueError, TypeError, SchemaError, RecursionError): + # Third-party schema catalogs can include dialects/extensions outside + # the local JSON Schema validator. Those cases are explicitly excluded. + return None + + +class PublicParquetBenchmark: + def __init__(self, spec: DatasetSpec) -> None: + self._spec = spec + + @property + def spec(self) -> DatasetSpec: + return self._spec + + def _convert(self, rows: list[dict]) -> tuple[list[Sample], dict]: + source = self.spec.source + samples: list[Sample] = [] + discarded = 0 + seen: set[str] = set() + max_schema_chars = int(self.spec.options.get("max_schema_chars", 12000)) + + for index, row in enumerate(rows): + common = { + "dataset_id": self.spec.dataset_id, + "dataset_revision": self.spec.revision, + "source_split": self.spec.split, + "source_index": index, + "provenance": "public-upstream", + } + if source == "openai/gsm8k": + question = _required_text(row.get("question"), case=str(index), field="question") + solution = _required_text(row.get("answer"), case=str(index), field="answer") + sample_id = f"gsm8k-main-test-{index:05d}" + value = Sample( + sample_id=sample_id, input=question, + expected=_gsm_final_answer(solution), + metadata={**common, "family": "grade_school_word_problem"}, + ) + elif source == "rajpurkar/squad_v2": + source_id = _required_text(row.get("id"), case=str(index), field="id") + context = _required_text(row.get("context"), case=source_id, field="context") + question = _required_text(row.get("question"), case=source_id, field="question") + aliases = _squad_answers(row.get("answers")) + answerable = bool(aliases) + sample_id = f"squad-v2-validation-{source_id}" + value = Sample( + sample_id=sample_id, + input={"context": context, "question": question}, + expected={"answer": aliases[0] if answerable else "", "answerable": answerable}, + metadata={ + **common, "answerable": answerable, + "family": "answerable" if answerable else "unanswerable", + "accepted_answers": list(aliases), + "answer_normalization": "squad-v2", + "source_title": str(row.get("title") or ""), + }, + ) + elif source == "epfl-dlab/JSONSchemaBench": + source_id = _required_text(row.get("unique_id"), case=str(index), field="unique_id") + schema = _schema(row.get("json_schema"), max_chars=max_schema_chars) + if schema is None: + discarded += 1 + continue + subset = str(self.spec.options.get("subset") or "Github_easy") + sample_id = f"jsonschemabench-{subset}-{self.spec.split}-{source_id}" + value = Sample( + sample_id=sample_id, + input="Generate an example JSON object satisfying the supplied schema.", + expected=None, # No semantic gold: this is a compliance task. + metadata={ + **common, "family": subset, + "response_schema": schema, + "schema_characters": len(row["json_schema"]), + "schema_root_type": "object", + }, + ) + else: + raise ValueError(f"Unknown public benchmark source {source!r}") + if sample_id in seen: + raise ValueError(f"Duplicate public benchmark sample ID: {sample_id}") + seen.add(sample_id) + samples.append(value) + return samples, {"source_rows": len(rows), "excluded_rows": discarded} + + def load(self, context: DatasetLoadContext) -> DatasetLoadResult: + path = single_cached_source(self.spec, context) + columns = { + "openai/gsm8k": ("question", "answer"), + "rajpurkar/squad_v2": ("id", "title", "context", "question", "answers"), + "epfl-dlab/JSONSchemaBench": ("json_schema", "unique_id"), + }[self.spec.source] + available, stats = self._convert(_rows(path, columns)) + selected = select_samples( + available, dataset_id=self.spec.dataset_id, context=context + ) + return DatasetLoadResult( + spec=self.spec, + samples=selected, + available_count=len(available), + selection_fingerprint=fingerprint_values([s.sample_id for s in selected]), + source_checksums={"source": sha256_file(path)}, + metadata={ + **stats, "eligible_rows": len(available), + "selection_strategy": str(context.selection.get("strategy") or "profile"), + "source": self.spec.source, + "source_revision": self.spec.revision, + "source_split": self.spec.split, + "license": self.spec.license_id, + "subset": self.spec.options.get("subset"), + }, + ) diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py index bebd5004..8a0c41f0 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py @@ -11,6 +11,7 @@ from model_capability_bench.datasets.banking77 import Banking77Dataset from model_capability_bench.datasets.clinc150 import Clinc150OosDataset from model_capability_bench.datasets.controlled_yaml import ControlledYamlDataset +from model_capability_bench.datasets.public_benchmarks import PublicParquetBenchmark from model_capability_bench.datasets.structured_output import ( StructuredOutputControlledDataset, ) @@ -25,6 +26,7 @@ def build_dataset_plugins() -> DatasetPluginRegistry: StructuredOutputControlledDataset, ) plugins.register("controlled-yaml", ControlledYamlDataset) + plugins.register("public-parquet", PublicParquetBenchmark) return plugins diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py b/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py index aebbc400..a1405033 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py @@ -30,12 +30,14 @@ ) -def _config_checksums(root: Path) -> dict[str, str]: +def _config_checksums(root: Path, *, suite_filename: str = "suite.yaml") -> dict[str, str]: checksums: dict[str, str] = {} for name in CONFIG_FILES: path = root / name if path.is_file(): checksums[name] = sha256_file(path) + if suite_filename not in checksums: + checksums[suite_filename] = sha256_file(root / suite_filename) return checksums @@ -182,6 +184,7 @@ def write_run_artifacts( "suite": { "suite_id": suite.suite.suite_id, "version": suite.suite.version, + "suite_file": suite.suite_filename, "profile": config.profile, "seed": config.seed, }, @@ -200,7 +203,9 @@ def write_run_artifacts( }, "capabilities": selected_capabilities, "signatures": signatures, - "config_checksums": _config_checksums(suite.root), + "config_checksums": _config_checksums( + suite.root, suite_filename=suite.suite_filename + ), "evidence": { "state": "state.jsonl", "raw": "raw.jsonl", diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py index 621c7b66..d29d8540 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py @@ -50,6 +50,7 @@ class CapabilitySuiteBundle: tasks: TaskRegistry datasets: DatasetRegistry profiles: Mapping[str, DatasetProfileSpec] + suite_filename: str = "suite.yaml" def plan_matrix( self, @@ -99,13 +100,17 @@ def plan_matrix( return tuple(arms) -def load_capability_suite(root: str | Path) -> CapabilitySuiteBundle: +def load_capability_suite( + root: str | Path, *, suite_filename: str = "suite.yaml" +) -> CapabilitySuiteBundle: + if Path(suite_filename).name != suite_filename or not suite_filename.endswith(".yaml"): + raise ValueError("suite_filename must be a YAML basename within the benchmark root") root_path = Path(root) models = load_registry(root_path / "models.yaml") tasks = build_task_registry(root_path / "tasks.yaml") datasets = build_dataset_registry(root_path / "datasets.yaml") profiles = load_dataset_profiles(root_path / "profiles.yaml") - suite = load_suite_spec(root_path / "suite.yaml") + suite = load_suite_spec(root_path / suite_filename) resolved = validate_suite( suite, tasks=tasks, @@ -120,4 +125,5 @@ def load_capability_suite(root: str | Path) -> CapabilitySuiteBundle: tasks=tasks, datasets=datasets, profiles=profiles, + suite_filename=suite_filename, ) diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py index c4b55305..05717205 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py @@ -5,11 +5,13 @@ from model_capability_bench.tasks.qa_abstention import QaAbstentionTask from model_capability_bench.tasks.reasoning import MathematicalReasoningTask from model_capability_bench.tasks.registry import build_task_plugins, build_task_registry +from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask from model_capability_bench.tasks.structured_output import StructuredOutputTask __all__ = [ "CalibratedIntentClassificationTask", "IntentClassificationTask", + "JsonSchemaComplianceTask", "MathematicalReasoningTask", "QaAbstentionTask", "StructuredOutputTask", diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py index 732aeb7a..8f5f16c1 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re +import string from collections.abc import Mapping from typing import Any @@ -49,6 +51,27 @@ def _expected(sample: Sample) -> tuple[str, bool]: def _normalize_answer(value: str) -> str: return " ".join(value.strip().lower().split()) + @staticmethod + def _answer_matches(sample: Sample, prediction: str, expected: str) -> bool: + """Preserve legacy exact semantics; SQuAD uses its published normalization.""" + if sample.metadata.get("answer_normalization") != "squad-v2": + return QaAbstentionTask._normalize_answer(prediction) == ( + QaAbstentionTask._normalize_answer(expected) + ) + aliases = sample.metadata.get("accepted_answers") + if not isinstance(aliases, (list, tuple)) or not aliases: + aliases = [expected] + + def squad_normalize(value: str) -> str: + lowered = value.lower() + no_punctuation = "".join(c for c in lowered if c not in string.punctuation) + no_articles = re.sub(r"\b(a|an|the)\b", " ", no_punctuation) + return " ".join(no_articles.split()) + + return squad_normalize(prediction) in { + squad_normalize(alias) for alias in aliases if isinstance(alias, str) + } + def build_request( self, sample: Sample, @@ -106,8 +129,7 @@ def evaluate( exact_match: float | None if answerable and valid and not abstain and predicted_answer is not None: exact_match = float( - self._normalize_answer(predicted_answer) - == self._normalize_answer(expected_answer) + self._answer_matches(sample, predicted_answer, expected_answer) ) elif answerable: exact_match = 0.0 diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py index 47e223fd..da0b4e2b 100644 --- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py +++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py @@ -10,6 +10,7 @@ ) from model_capability_bench.tasks.qa_abstention import QaAbstentionTask from model_capability_bench.tasks.reasoning import MathematicalReasoningTask +from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask from model_capability_bench.tasks.structured_output import StructuredOutputTask @@ -23,6 +24,7 @@ def build_task_plugins() -> TaskPluginRegistry: plugins.register("qa-abstention", QaAbstentionTask) plugins.register("mathematical-reasoning", MathematicalReasoningTask) plugins.register("structured-output", StructuredOutputTask) + plugins.register("json-schema-compliance", JsonSchemaComplianceTask) return plugins diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py new file mode 100644 index 00000000..afb82343 --- /dev/null +++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py @@ -0,0 +1,92 @@ +"""Schema compliance is distinct from semantic field extraction. + +JSONSchemaBench contains schemas, not source texts with gold field values. +Therefore only validity against the upstream schema is scored here; this +does not attempt to reproduce the official JSONSchemaBench harness score. +""" +from __future__ import annotations + +import json +from collections.abc import Mapping + +from benchmark_core import ( + InferenceRequest, + InferenceResult, + MetricResult, + Sample, + TaskExecutionContext, + TaskResult, + TaskSpec, +) +from jsonschema.validators import validator_for + + +class JsonSchemaComplianceTask: + def __init__(self, spec: TaskSpec) -> None: + self._spec = spec + + @property + def spec(self) -> TaskSpec: + return self._spec + + def _schema(self, sample: Sample) -> Mapping: + schema = sample.metadata.get("response_schema") + if not isinstance(schema, Mapping): + raise TypeError(f"{sample.sample_id}: missing response_schema") + validator_for(schema).check_schema(schema) + return schema + + def build_request( + self, sample: Sample, context: TaskExecutionContext + ) -> InferenceRequest: + schema = self._schema(sample) + return InferenceRequest( + request_id=f"{context.run_id}:{self.spec.task_id}:{sample.sample_id}", + input=sample.input, + system_prompt=str(self.spec.options.get("instruction") or + "Return only a JSON object conforming to the schema."), + response_schema=schema, + generation=context.generation, + task_metadata={ + "task_id": self.spec.task_id, + "task_version": self.spec.version, + "prompt_id": self.spec.prompt_id, + "prompt_version": self.spec.prompt_version, + "dataset_id": context.dataset_id, + }, + ) + + def evaluate( + self, sample: Sample, inference: InferenceResult, + context: TaskExecutionContext, + ) -> TaskResult: + prediction = inference.normalized_output + if isinstance(prediction, str): + try: + prediction = json.loads(prediction) + except json.JSONDecodeError: + pass + parse_valid = isinstance(prediction, dict) + schema = self._schema(sample) + schema_valid = bool( + inference.valid + and parse_valid + and validator_for(schema)(schema).is_valid(prediction) + ) + error = inference.error.message if inference.error else None + if inference.valid and not parse_valid: + error = "normalized output must be a JSON object" + return TaskResult( + task_id=self.spec.task_id, + sample_id=sample.sample_id, + prediction=prediction, + expected=None, + metrics=( + MetricResult(name="schema_valid_rate", value=float(schema_valid), primary=True), + MetricResult(name="parse_valid", value=float(inference.valid and parse_valid)), + ), + valid=bool(inference.valid and parse_valid), + error=error, + metadata={"dataset_id": context.dataset_id, + "schema_compliance_only": True}, + ) diff --git a/experiments/model-capability-benchmark/suite_public.yaml b/experiments/model-capability-benchmark/suite_public.yaml new file mode 100644 index 00000000..355ff7bd --- /dev/null +++ b/experiments/model-capability-benchmark/suite_public.yaml @@ -0,0 +1,140 @@ +suite: + id: capability-public-benchmarks + version: "1" + description: >- + Opt-in pinned public test/validation subsets. These MCB-specific protocols + are not official full-benchmark leaderboard scores. + default_profile: smoke + generation: + temperature: 0.0 + max_output_tokens: 2048 + seed: 42 + + capabilities: + public-mathematical-reasoning: + task: mathematical-reasoning + datasets: [gsm8k-main-test] + description: GSM8K main/test numeric final-answer accuracy, MCB 0-shot JSON protocol. + tags: [math, gsm8k, public, subset] + benchmark: + smoke: + max_cases: 12 + target_local_seconds: 180 + hard_local_seconds: 300 + core: + max_cases: 40 + target_local_seconds: 600 + hard_local_seconds: 900 + extended: + max_cases: 200 + hard_local_seconds: 3600 + comparison: + practical_delta: 0.05 + metric: final_answer_accuracy + metrics: + - name: final_answer_accuracy + source: task_metric + reducer: mean + primary: true + - name: parse_valid + source: task_metric + reducer: mean + - name: latency_p95_ms + source: inference + reducer: p95 + field: latency_ms + + public-qa-abstention: + task: qa-abstention + datasets: [squad-v2-validation] + description: SQuAD 2.0 validation; MCB JSON answer/abstain protocol, alias-aware exact match. + tags: [qa, abstention, squad-v2, public, subset] + benchmark: + smoke: + max_cases: 12 + selection: + strategy: stratified + quotas: + - match: {answerable: true} + count: 6 + - match: {answerable: false} + count: 6 + target_local_seconds: 180 + hard_local_seconds: 300 + core: + max_cases: 60 + selection: + strategy: stratified + quotas: + - match: {answerable: true} + count: 30 + - match: {answerable: false} + count: 30 + target_local_seconds: 600 + hard_local_seconds: 900 + extended: + max_cases: 200 + selection: + strategy: stratified + quotas: + - match: {answerable: true} + count: 100 + - match: {answerable: false} + count: 100 + hard_local_seconds: 3600 + comparison: + practical_delta: 0.05 + metric: qa_correct + metrics: + - name: qa_correct + source: task_metric + reducer: mean + primary: true + - name: answerability_accuracy + source: task_metric + reducer: mean + - name: exact_match + source: task_metric + reducer: mean + - name: parse_valid + source: task_metric + reducer: mean + - name: latency_p95_ms + source: inference + reducer: p95 + field: latency_ms + + public-json-schema-compliance: + task: json-schema-compliance + datasets: [jsonschemabench-github-easy-test] + description: >- + JSONSchemaBench Github_easy/test; object-root, locally-resolvable, bounded + schemas only. Compliance scoring, not semantic extraction or official harness. + tags: [structured-output, json-schema, public, subset] + benchmark: + smoke: + max_cases: 12 + target_local_seconds: 180 + hard_local_seconds: 300 + core: + max_cases: 40 + target_local_seconds: 600 + hard_local_seconds: 900 + extended: + max_cases: 150 + hard_local_seconds: 3600 + comparison: + practical_delta: 0.05 + metric: schema_valid_rate + metrics: + - name: schema_valid_rate + source: task_metric + reducer: mean + primary: true + - name: parse_valid + source: task_metric + reducer: mean + - name: latency_p95_ms + source: inference + reducer: p95 + field: latency_ms diff --git a/experiments/model-capability-benchmark/tasks.yaml b/experiments/model-capability-benchmark/tasks.yaml index d6e89948..89d15204 100644 --- a/experiments/model-capability-benchmark/tasks.yaml +++ b/experiments/model-capability-benchmark/tasks.yaml @@ -86,6 +86,7 @@ tasks: compatible_datasets: - qa-abstention-controlled-v1 - qa-abstention-controlled-v2 + - squad-v2-validation required_capabilities: text_input: true prompt: @@ -112,6 +113,7 @@ tasks: - math-reasoning-controlled-v1 - math-reasoning-controlled-v2 - math-reasoning-controlled-v3 + - gsm8k-main-test required_capabilities: text_input: true prompt: @@ -126,3 +128,24 @@ tasks: Solve the problem and return only the final answer in the requested structured field. Do not include private chain-of-thought or intermediate reasoning. output_field: answer + + json-schema-compliance: + version: "1" + plugin: json-schema-compliance + evaluator: json-schema-validation + evaluator_version: "1" + compatible_datasets: + - jsonschemabench-github-easy-test + required_capabilities: + text_input: true + prompt: + id: json-schema-compliance-v1 + version: "1" + metrics: + - name: schema_valid_rate + primary: true + - name: parse_valid + options: + instruction: >- + Generate an example JSON object satisfying the provided JSON Schema. + Return only the JSON object and do not include explanatory text. diff --git a/experiments/model-capability-benchmark/tests/test_dataset_adapters.py b/experiments/model-capability-benchmark/tests/test_dataset_adapters.py index f11e5e38..14590096 100644 --- a/experiments/model-capability-benchmark/tests/test_dataset_adapters.py +++ b/experiments/model-capability-benchmark/tests/test_dataset_adapters.py @@ -69,15 +69,18 @@ def test_catalog_profiles_and_task_references_are_consistent() -> None: profiles = load_dataset_profiles(PROFILES) assert datasets.summary() == { - "datasets": 9, + "datasets": 12, "dataset_ids": [ "banking77", "clinc150-oos", + "gsm8k-main-test", + "jsonschemabench-github-easy-test", "math-reasoning-controlled-v1", "math-reasoning-controlled-v2", "math-reasoning-controlled-v3", "qa-abstention-controlled-v1", "qa-abstention-controlled-v2", + "squad-v2-validation", "structured-output-controlled-v1", "structured-output-controlled-v2", ], @@ -85,6 +88,7 @@ def test_catalog_profiles_and_task_references_are_consistent() -> None: "banking77", "clinc150-oos", "controlled-yaml", + "public-parquet", "structured-output-controlled", ], } @@ -192,7 +196,6 @@ def test_profiles_are_configuration_driven() -> None: def test_public_dataset_catalog_uses_pinned_revisions_and_no_moving_main() -> None: - text = DATASETS.read_text(encoding="utf-8") registry = build_dataset_registry(DATASETS) assert registry.get("banking77").spec.revision == ( @@ -201,8 +204,19 @@ def test_public_dataset_catalog_uses_pinned_revisions_and_no_moving_main() -> No assert registry.get("clinc150-oos").spec.revision == ( "48a0e1cff8f43dd4d0836ecb4ed5df08733e3d2e" ) - assert "/main/" not in text - assert "refs/heads/main" not in text + for dataset in registry.select(): + if dataset.spec.cache_mode != "download": + continue + files = dataset.spec.options.get("files") or { + "source": dataset.spec.options.get("file") + } + for entry in files.values(): + url = entry["url"] + if "huggingface.co/" in url: + assert f"/resolve/{dataset.spec.revision}/" in url + else: + assert f"/{dataset.spec.revision}/" in url + assert "refs/heads/main" not in url def test_v2_controlled_datasets_have_vertical_family_coverage(tmp_path: Path) -> None: diff --git a/experiments/model-capability-benchmark/tests/test_public_benchmarks.py b/experiments/model-capability-benchmark/tests/test_public_benchmarks.py new file mode 100644 index 00000000..51f760bf --- /dev/null +++ b/experiments/model-capability-benchmark/tests/test_public_benchmarks.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from benchmark_core import ( + InferenceResult, + TaskExecutionContext, +) + +from model_capability_bench import build_dataset_registry, load_capability_suite +from model_capability_bench.datasets import public_benchmarks +from model_capability_bench.runner.capability_data import load_capability_datasets + +ROOT = Path(__file__).resolve().parents[1] + + +def _fixture_spec(dataset_id: str): + return build_dataset_registry(ROOT / "datasets.yaml").get(dataset_id).spec + + +def test_opt_in_suite_does_not_change_legacy_core() -> None: + old = load_capability_suite(ROOT) + assert old.suite.suite_id == "capability-core" + assert len(old.resolved_capabilities) == 5 + + public = load_capability_suite(ROOT, suite_filename="suite_public.yaml") + assert public.suite.suite_id == "capability-public-benchmarks" + assert len(public.resolved_capabilities) == 3 + assert public.suite_filename == "suite_public.yaml" + assert {c.spec.capability_id for c in public.resolved_capabilities} == { + "public-mathematical-reasoning", + "public-qa-abstention", + "public-json-schema-compliance", + } + + +def test_gsm8k_gold_final_answers_are_numeric_and_source_rows_preserved() -> None: + adapter = public_benchmarks.PublicParquetBenchmark(_fixture_spec("gsm8k-main-test")) + rows = [ + {"question": "If a pack costs $1, how much do 1,200 packs cost?", + "answer": "Compute the total.\n#### 1,200"}, + {"question": "Find the remaining balance.", + "answer": "Subtract amounts.\n#### 3.50"}, + ] + samples, stats = adapter._convert(rows) + assert [case.expected for case in samples] == ["1200", "3.5"] + assert [case.metadata["source_index"] for case in samples] == [0, 1] + assert len({case.sample_id for case in samples}) == 2 + assert stats == {"source_rows": 2, "excluded_rows": 0} + + +def test_squad_gold_aliases_and_abstention_preserved() -> None: + adapter = public_benchmarks.PublicParquetBenchmark( + _fixture_spec("squad-v2-validation") + ) + rows = [ + {"id": "abc1", "title": "Example", "context": "The answer is the 10th century.", + "question": "When?", "answers": { + "text": ["the 10th century", "10th century"], + "answer_start": [14, 18], + }}, + {"id": "abc2", "title": "Example", "context": "It does not say.", + "question": "When?", "answers": {"text": [], "answer_start": []}}, + ] + samples, _ = adapter._convert(rows) + assert samples[0].metadata["accepted_answers"] == [ + "the 10th century", "10th century" + ] + assert samples[1].expected == {"answer": "", "answerable": False} + + suite = load_capability_suite(ROOT, suite_filename="suite_public.yaml") + task = suite.tasks.get("qa-abstention") + context = TaskExecutionContext( + run_id="public-test", dataset_id="squad-v2-validation", + profile="core", generation=suite.suite.generation, + ) + correct = task.evaluate( + samples[0], + InferenceResult( + provider_id="test", model_id="test", + raw_output={"answer": "10th century", "abstain": False}, + normalized_output={"answer": "10th century", "abstain": False}, + latency_ms=0.0, + ), + context, + ) + assert next(m.value for m in correct.metrics if m.name == "qa_correct") == 1.0 + + abstained = task.evaluate( + samples[1], + InferenceResult( + provider_id="test", model_id="test", + raw_output={"answer": "", "abstain": True}, + normalized_output={"answer": "", "abstain": True}, + latency_ms=0.0, + ), + context, + ) + assert next(m.value for m in abstained.metrics if m.name == "qa_correct") == 1.0 + + +def test_squad_core_selects_balanced_answerability_without_network( + monkeypatch, tmp_path: Path +) -> None: + rows = [ + {"id": f"{n:05d}", "title": "Fixture", + "context": f"Person {n} lives in Rome.", + "question": f"Where does person {n} live?", + "answers": { + "text": ["Rome"] if n % 2 else [], + "answer_start": [18] if n % 2 else [], + }} + for n in range(160) + ] + path = tmp_path / "fixture.parquet" + path.write_bytes(b"test-only data") + monkeypatch.setattr(public_benchmarks, "single_cached_source", lambda *_: path) + monkeypatch.setattr(public_benchmarks, "_rows", lambda *_: rows) + bundle = load_capability_suite(ROOT, suite_filename="suite_public.yaml") + capability = next( + c for c in bundle.resolved_capabilities + if c.spec.capability_id == "public-qa-abstention" + ) + kwargs = { + "suite": bundle, "capability": capability, + "profile": bundle.profiles["core"], "profile_id": "core", + "cache_dir": tmp_path, "seed": 42, + } + a = load_capability_datasets(**kwargs, cache={})["squad-v2-validation"] + b = load_capability_datasets(**kwargs, cache={})["squad-v2-validation"] + assert len(a.samples) == 60 + assert a.selection_fingerprint == b.selection_fingerprint + assert Counter(c.metadata["answerable"] for c in a.samples) == { + True: 30, False: 30 + } + assert set(a.source_checksums) == {"source"} + + +def test_jsonschemabench_only_scores_schema_compliance() -> None: + adapter = public_benchmarks.PublicParquetBenchmark( + _fixture_spec("jsonschemabench-github-easy-test") + ) + schema = { + "type": "object", + "properties": {"count": {"type": "integer", "minimum": 2}}, + "required": ["count"], "additionalProperties": False, + } + remote = {"type": "object", "properties": { + "external": {"$ref": "https://example.org/anything"} + }} + cases, stats = adapter._convert([ + {"unique_id": "ok", "json_schema": json.dumps(schema)}, + {"unique_id": "remote", "json_schema": json.dumps(remote)}, + ]) + assert len(cases) == 1 + assert stats["excluded_rows"] == 1 + assert cases[0].expected is None + + suite = load_capability_suite(ROOT, suite_filename="suite_public.yaml") + task = suite.tasks.get("json-schema-compliance") + ctx = TaskExecutionContext( + run_id="public-test", dataset_id="jsonschemabench-github-easy-test", + profile="core", generation=suite.suite.generation, + ) + + def score(value): + result = task.evaluate( + cases[0], + InferenceResult(provider_id="test", model_id="test", + raw_output=value, normalized_output=value, + latency_ms=0.0), + ctx, + ) + return next(m.value for m in result.metrics + if m.name == "schema_valid_rate") + + assert score({"count": 3}) == 1.0 + assert score({"count": 1}) == 0.0 + assert score({"count": "3"}) == 0.0 + + +def test_public_suite_path_is_restricted() -> None: + import pytest + + with pytest.raises(ValueError, match="YAML basename"): + load_capability_suite(ROOT, suite_filename="../other.yaml") diff --git a/experiments/model-capability-benchmark/tests/test_task_plugins.py b/experiments/model-capability-benchmark/tests/test_task_plugins.py index 150c3f32..3c902a34 100644 --- a/experiments/model-capability-benchmark/tests/test_task_plugins.py +++ b/experiments/model-capability-benchmark/tests/test_task_plugins.py @@ -29,10 +29,11 @@ def test_task_catalog_builds_without_runner_task_branches() -> None: registry = build_task_registry(TASKS) assert registry.summary() == { - "tasks": 5, + "tasks": 6, "task_ids": [ "calibrated-intent-classification", "intent-classification", + "json-schema-compliance", "mathematical-reasoning", "qa-abstention", "structured-output", @@ -40,6 +41,7 @@ def test_task_catalog_builds_without_runner_task_branches() -> None: "plugins": [ "calibrated-intent-classification", "intent-classification", + "json-schema-compliance", "mathematical-reasoning", "qa-abstention", "structured-output",