From eea8f918f0e69fcd3591075b34dbcb3a8126a018 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:43:58 +0200
Subject: [PATCH 01/33] feat(mcb): add pinned public Parquet benchmark adapters

---
 .../datasets/public_benchmarks.py             | 202 ++++++++++++++++++
 1 file changed, 202 insertions(+)
 create mode 100644 experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
new file mode 100644
index 00000000..94c2e61d
--- /dev/null
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
@@ -0,0 +1,202 @@
+"""Pinned, offline-cacheable public benchmark Parquet adapters.
+
+The original test/validation rows and gold answers are not modified. MCB selects a
+small deterministic subset, and records the exact upstream artifact checksum.
+These are MCB subsets, not claims of official full-benchmark scores.
+"""
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Mapping
+from decimal import Decimal, InvalidOperation
+
+from benchmark_core import (
+    DatasetLoadContext,
+    DatasetLoadResult,
+    DatasetSpec,
+    Sample,
+    fingerprint_values,
+    sha256_file,
+)
+from jsonschema.validators import validator_for
+
+from model_capability_bench.datasets.common import single_cached_source
+from model_capability_bench.datasets.selection import select_samples
+
+
+def _rows(path, columns: tuple[str, ...]) -> list[dict]:
+    try:
+        import pyarrow.parquet as pq
+    except ImportError as exc:
+        raise RuntimeError(
+            "Public Parquet datasets require pyarrow; install MCB dependencies"
+        ) from exc
+    return pq.read_table(path, columns=list(columns)).to_pylist()
+
+
+def _required_text(value: object, *, case: str, field: str) -> str:
+    if not isinstance(value, str) or not value.strip():
+        raise ValueError(f"{case}: missing or empty {field}")
+    return value
+
+
+def _gsm_final_answer(solution: str) -> str:
+    match = re.search(r"(?m)^\s*####\s*([^\r\n]+)\s*$", solution)
+    if not match:
+        raise ValueError("GSM8K gold has no #### final answer")
+    text = match.group(1).strip().replace(",", "")
+    try:
+        result = Decimal(text)
+    except InvalidOperation as exc:
+        raise ValueError(f"Non-numeric GSM8K final answer: {text!r}") from exc
+    if not result.is_finite():
+        raise ValueError("Non-finite GSM8K final answer")
+    return format(result.normalize(), "f")
+
+
+def _squad_answers(value: object) -> tuple[str, ...]:
+    if not isinstance(value, Mapping):
+        raise ValueError("SQuAD answers must be an object")
+    texts = value.get("text")
+    if not isinstance(texts, list) or not all(isinstance(t, str) for t in texts):
+        raise ValueError("SQuAD answers.text must be a string list")
+    return tuple(dict.fromkeys(texts))
+
+
+def _unsafe_ref(value: object) -> bool:
+    if isinstance(value, Mapping):
+        for key, item in value.items():
+            if key in {"$ref", "$dynamicRef", "$recursiveRef"}:
+                # Avoid implicit network access and runtime-dependent reference resolution.
+                if not isinstance(item, str) or not item.startswith("#/"):
+                    return True
+            if _unsafe_ref(item):
+                return True
+    elif isinstance(value, list):
+        return any(_unsafe_ref(item) for item in value)
+    return False
+
+
+def _schema(value: object, *, max_chars: int) -> dict | None:
+    if not isinstance(value, str) or len(value) > max_chars:
+        return None
+    try:
+        schema = json.loads(value)
+        if not isinstance(schema, dict) or schema.get("type") != "object":
+            return None
+        if _unsafe_ref(schema):
+            return None
+        validator_for(schema).check_schema(schema)
+        return schema
+    except Exception:
+        # Third-party schema catalogs can include dialects/extensions outside
+        # the local JSON Schema validator. Those cases are explicitly excluded.
+        return None
+
+
+class PublicParquetBenchmark:
+    def __init__(self, spec: DatasetSpec) -> None:
+        self._spec = spec
+
+    @property
+    def spec(self) -> DatasetSpec:
+        return self._spec
+
+    def _convert(self, rows: list[dict]) -> tuple[list[Sample], dict]:
+        source = self.spec.source
+        samples: list[Sample] = []
+        discarded = 0
+        seen: set[str] = set()
+        max_schema_chars = int(self.spec.options.get("max_schema_chars", 12000))
+
+        for index, row in enumerate(rows):
+            common = {
+                "dataset_id": self.spec.dataset_id,
+                "dataset_revision": self.spec.revision,
+                "source_split": self.spec.split,
+                "source_index": index,
+                "provenance": "public-upstream",
+            }
+            if source == "openai/gsm8k":
+                question = _required_text(row.get("question"), case=str(index), field="question")
+                solution = _required_text(row.get("answer"), case=str(index), field="answer")
+                sample_id = f"gsm8k-main-test-{index:05d}"
+                value = Sample(
+                    sample_id=sample_id, input=question,
+                    expected=_gsm_final_answer(solution),
+                    metadata={**common, "family": "grade_school_word_problem"},
+                )
+            elif source == "rajpurkar/squad_v2":
+                source_id = _required_text(row.get("id"), case=str(index), field="id")
+                context = _required_text(row.get("context"), case=source_id, field="context")
+                question = _required_text(row.get("question"), case=source_id, field="question")
+                aliases = _squad_answers(row.get("answers"))
+                answerable = bool(aliases)
+                sample_id = f"squad-v2-validation-{source_id}"
+                value = Sample(
+                    sample_id=sample_id,
+                    input={"context": context, "question": question},
+                    expected={"answer": aliases[0] if answerable else "", "answerable": answerable},
+                    metadata={
+                        **common, "answerable": answerable,
+                        "family": "answerable" if answerable else "unanswerable",
+                        "accepted_answers": list(aliases),
+                        "answer_normalization": "squad-v2",
+                        "source_title": str(row.get("title") or ""),
+                    },
+                )
+            elif source == "epfl-dlab/JSONSchemaBench":
+                source_id = _required_text(row.get("unique_id"), case=str(index), field="unique_id")
+                schema = _schema(row.get("json_schema"), max_chars=max_schema_chars)
+                if schema is None:
+                    discarded += 1
+                    continue
+                subset = str(self.spec.options.get("subset") or "Github_easy")
+                sample_id = f"jsonschemabench-{subset}-{self.spec.split}-{source_id}"
+                value = Sample(
+                    sample_id=sample_id,
+                    input="Generate an example JSON object satisfying the supplied schema.",
+                    expected=None,  # No semantic gold: this is a compliance task.
+                    metadata={
+                        **common, "family": subset,
+                        "response_schema": schema,
+                        "schema_characters": len(row["json_schema"]),
+                        "schema_root_type": "object",
+                    },
+                )
+            else:
+                raise ValueError(f"Unknown public benchmark source {source!r}")
+            if sample_id in seen:
+                raise ValueError(f"Duplicate public benchmark sample ID: {sample_id}")
+            seen.add(sample_id)
+            samples.append(value)
+        return samples, {"source_rows": len(rows), "excluded_rows": discarded}
+
+    def load(self, context: DatasetLoadContext) -> DatasetLoadResult:
+        path = single_cached_source(self.spec, context)
+        columns = {
+            "openai/gsm8k": ("question", "answer"),
+            "rajpurkar/squad_v2": ("id", "title", "context", "question", "answers"),
+            "epfl-dlab/JSONSchemaBench": ("json_schema", "unique_id"),
+        }[self.spec.source]
+        available, stats = self._convert(_rows(path, columns))
+        selected = select_samples(
+            available, dataset_id=self.spec.dataset_id, context=context
+        )
+        return DatasetLoadResult(
+            spec=self.spec,
+            samples=selected,
+            available_count=len(available),
+            selection_fingerprint=fingerprint_values([s.sample_id for s in selected]),
+            source_checksums={"source": sha256_file(path)},
+            metadata={
+                **stats, "eligible_rows": len(available),
+                "selection_strategy": str(context.selection.get("strategy") or "profile"),
+                "source": self.spec.source,
+                "source_revision": self.spec.revision,
+                "source_split": self.spec.split,
+                "license": self.spec.license_id,
+                "subset": self.spec.options.get("subset"),
+            },
+        )

From 748e0410e0afe4f24db4d9cb4a6c788c93ffa41e Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:44:18 +0200
Subject: [PATCH 02/33] feat(mcb): add JSON schema compliance task distinct
 from extraction

---
 .../tasks/schema_compliance.py                | 92 +++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py
new file mode 100644
index 00000000..4fe0994b
--- /dev/null
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py
@@ -0,0 +1,92 @@
+"""Schema compliance is distinct from semantic field extraction.
+
+JSONSchemaBench contains schemas, not source texts with gold field values.
+Therefore only validity against the upstream schema is scored here; this
+does not attempt to reproduce the official JSONSchemaBench harness score.
+"""
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+
+from benchmark_core import (
+    InferenceRequest,
+    InferenceResult,
+    MetricResult,
+    Sample,
+    TaskExecutionContext,
+    TaskResult,
+    TaskSpec,
+)
+from jsonschema.validators import validator_for
+
+
+class JsonSchemaComplianceTask:
+    def __init__(self, spec: TaskSpec) -> None:
+        self._spec = spec
+
+    @property
+    def spec(self) -> TaskSpec:
+        return self._spec
+
+    def _schema(self, sample: Sample) -> Mapping:
+        schema = sample.metadata.get("response_schema")
+        if not isinstance(schema, Mapping):
+            raise ValueError(f"{sample.sample_id}: missing response_schema")
+        validator_for(schema).check_schema(schema)
+        return schema
+
+    def build_request(
+        self, sample: Sample, context: TaskExecutionContext
+    ) -> InferenceRequest:
+        schema = self._schema(sample)
+        return InferenceRequest(
+            request_id=f"{context.run_id}:{self.spec.task_id}:{sample.sample_id}",
+            input=sample.input,
+            system_prompt=str(self.spec.options.get("instruction") or
+                              "Return only a JSON object conforming to the schema."),
+            response_schema=schema,
+            generation=context.generation,
+            task_metadata={
+                "task_id": self.spec.task_id,
+                "task_version": self.spec.version,
+                "prompt_id": self.spec.prompt_id,
+                "prompt_version": self.spec.prompt_version,
+                "dataset_id": context.dataset_id,
+            },
+        )
+
+    def evaluate(
+        self, sample: Sample, inference: InferenceResult,
+        context: TaskExecutionContext,
+    ) -> TaskResult:
+        prediction = inference.normalized_output
+        if isinstance(prediction, str):
+            try:
+                prediction = json.loads(prediction)
+            except json.JSONDecodeError:
+                pass
+        parse_valid = isinstance(prediction, dict)
+        schema = self._schema(sample)
+        schema_valid = bool(
+            inference.valid
+            and parse_valid
+            and validator_for(schema)(schema).is_valid(prediction)
+        )
+        error = inference.error.message if inference.error else None
+        if inference.valid and not parse_valid:
+            error = "normalized output must be a JSON object"
+        return TaskResult(
+            task_id=self.spec.task_id,
+            sample_id=sample.sample_id,
+            prediction=prediction,
+            expected=None,
+            metrics=(
+                MetricResult(name="schema_valid_rate", value=float(schema_valid), primary=True),
+                MetricResult(name="parse_valid", value=float(inference.valid and parse_valid)),
+            ),
+            valid=bool(inference.valid and parse_valid),
+            error=error,
+            metadata={"dataset_id": context.dataset_id,
+                      "schema_compliance_only": True},
+        )

From 380b7778445a4d06e25777649d1a3be314f0e821 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:44:50 +0200
Subject: [PATCH 03/33] feat(mcb): register public parquet datasets

---
 .../src/model_capability_bench/datasets/registry.py             | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py
index bebd5004..8a0c41f0 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/registry.py
@@ -11,6 +11,7 @@
 from model_capability_bench.datasets.banking77 import Banking77Dataset
 from model_capability_bench.datasets.clinc150 import Clinc150OosDataset
 from model_capability_bench.datasets.controlled_yaml import ControlledYamlDataset
+from model_capability_bench.datasets.public_benchmarks import PublicParquetBenchmark
 from model_capability_bench.datasets.structured_output import (
     StructuredOutputControlledDataset,
 )
@@ -25,6 +26,7 @@ def build_dataset_plugins() -> DatasetPluginRegistry:
         StructuredOutputControlledDataset,
     )
     plugins.register("controlled-yaml", ControlledYamlDataset)
+    plugins.register("public-parquet", PublicParquetBenchmark)
     return plugins
 
 

From c5da78ac6d7d71bdb2286b588365615aeb5dfe6e Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:44:53 +0200
Subject: [PATCH 04/33] feat(mcb): register schema compliance evaluator

---
 .../src/model_capability_bench/tasks/registry.py                | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py
index 47e223fd..4b82afde 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py
@@ -9,6 +9,7 @@
     IntentClassificationTask,
 )
 from model_capability_bench.tasks.qa_abstention import QaAbstentionTask
+from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask
 from model_capability_bench.tasks.reasoning import MathematicalReasoningTask
 from model_capability_bench.tasks.structured_output import StructuredOutputTask
 
@@ -23,6 +24,7 @@ def build_task_plugins() -> TaskPluginRegistry:
     plugins.register("qa-abstention", QaAbstentionTask)
     plugins.register("mathematical-reasoning", MathematicalReasoningTask)
     plugins.register("structured-output", StructuredOutputTask)
+    plugins.register("json-schema-compliance", JsonSchemaComplianceTask)
     return plugins
 
 

From 109266c9544ba8152c5221d0bac42b56d7aaca2d Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:44:56 +0200
Subject: [PATCH 05/33] feat(mcb): export public dataset adapter

---
 .../src/model_capability_bench/datasets/__init__.py             | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py
index aacae634..05639fb5 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/__init__.py
@@ -1,6 +1,7 @@
 from model_capability_bench.datasets.banking77 import Banking77Dataset
 from model_capability_bench.datasets.clinc150 import Clinc150OosDataset
 from model_capability_bench.datasets.controlled_yaml import ControlledYamlDataset
+from model_capability_bench.datasets.public_benchmarks import PublicParquetBenchmark
 from model_capability_bench.datasets.registry import (
     build_dataset_plugins,
     build_dataset_registry,
@@ -13,6 +14,7 @@
     "Banking77Dataset",
     "Clinc150OosDataset",
     "ControlledYamlDataset",
+    "PublicParquetBenchmark",
     "StructuredOutputControlledDataset",
     "build_dataset_plugins",
     "build_dataset_registry",

From f778bd2f81413d7c767119b59057f0ebff174d7f Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:44:58 +0200
Subject: [PATCH 06/33] feat(mcb): export JSON Schema task

---
 .../src/model_capability_bench/tasks/__init__.py                | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py
index c4b55305..0a931c09 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py
@@ -3,6 +3,7 @@
     IntentClassificationTask,
 )
 from model_capability_bench.tasks.qa_abstention import QaAbstentionTask
+from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask
 from model_capability_bench.tasks.reasoning import MathematicalReasoningTask
 from model_capability_bench.tasks.registry import build_task_plugins, build_task_registry
 from model_capability_bench.tasks.structured_output import StructuredOutputTask
@@ -12,6 +13,7 @@
     "IntentClassificationTask",
     "MathematicalReasoningTask",
     "QaAbstentionTask",
+    "JsonSchemaComplianceTask",
     "StructuredOutputTask",
     "build_task_plugins",
     "build_task_registry",

From 9defb57400a1589eaccc7bd2a8c7a935d9feafed Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:45:21 +0200
Subject: [PATCH 07/33] feat(mcb): pin GSM8K SQuAD2 JSONSchemaBench source
 artifacts

---
 .../model-capability-benchmark/datasets.yaml  | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/experiments/model-capability-benchmark/datasets.yaml b/experiments/model-capability-benchmark/datasets.yaml
index 277a8650..8f2764c0 100644
--- a/experiments/model-capability-benchmark/datasets.yaml
+++ b/experiments/model-capability-benchmark/datasets.yaml
@@ -156,3 +156,54 @@ datasets:
       path: data/math-reasoning-controlled-v3.yaml
       human_reviewed: false
       difficulty_labels: authored-provisional
+
+  # Public benchmarks: immutable Hugging Face dataset revisions and original test splits.
+  gsm8k-main-test:
+    version: "1"
+    adapter: public-parquet
+    source: openai/gsm8k
+    revision: 740312add88f781978c0658806c59bc2815b9866
+    split: test
+    license: MIT
+    sample_schema_version: "1"
+    cache_mode: download
+    options:
+      timeout_seconds: 120
+      user_agent: model-capability-bench/0.1
+      file:
+        url: https://huggingface.co/datasets/openai/gsm8k/resolve/740312add88f781978c0658806c59bc2815b9866/main/test-00000-of-00001.parquet
+        cache_path: gsm8k/740312add88f781978c0658806c59bc2815b9866/test.parquet
+
+  squad-v2-validation:
+    version: "1"
+    adapter: public-parquet
+    source: rajpurkar/squad_v2
+    revision: 3ffb306f725f7d2ce8394bc1873b24868140c412
+    split: validation
+    license: CC-BY-SA-4.0
+    sample_schema_version: "1"
+    cache_mode: download
+    options:
+      timeout_seconds: 120
+      user_agent: model-capability-bench/0.1
+      file:
+        url: https://huggingface.co/datasets/rajpurkar/squad_v2/resolve/3ffb306f725f7d2ce8394bc1873b24868140c412/squad_v2/validation-00000-of-00001.parquet
+        cache_path: squad-v2/3ffb306f725f7d2ce8394bc1873b24868140c412/validation.parquet
+
+  jsonschemabench-github-easy-test:
+    version: "1"
+    adapter: public-parquet
+    source: epfl-dlab/JSONSchemaBench
+    revision: 83df923c5b1dd05fb5c9659ad751d7c056a3cd43
+    split: test
+    license: MIT
+    sample_schema_version: "1"
+    cache_mode: download
+    options:
+      subset: Github_easy
+      max_schema_chars: 12000
+      timeout_seconds: 120
+      user_agent: model-capability-bench/0.1
+      file:
+        url: https://huggingface.co/datasets/epfl-dlab/JSONSchemaBench/resolve/83df923c5b1dd05fb5c9659ad751d7c056a3cd43/Github_easy/test-00000-of-00001.parquet
+        cache_path: jsonschemabench/83df923c5b1dd05fb5c9659ad751d7c056a3cd43/github_easy_test.parquet

From eaa339fd66992299446ac68f4252dca8264368bc Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:45:24 +0200
Subject: [PATCH 08/33] feat(mcb): wire public benchmarks to compatible task
 evaluators

---
 .../model-capability-benchmark/tasks.yaml     | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/experiments/model-capability-benchmark/tasks.yaml b/experiments/model-capability-benchmark/tasks.yaml
index d6e89948..89d15204 100644
--- a/experiments/model-capability-benchmark/tasks.yaml
+++ b/experiments/model-capability-benchmark/tasks.yaml
@@ -86,6 +86,7 @@ tasks:
     compatible_datasets:
       - qa-abstention-controlled-v1
       - qa-abstention-controlled-v2
+      - squad-v2-validation
     required_capabilities:
       text_input: true
     prompt:
@@ -112,6 +113,7 @@ tasks:
       - math-reasoning-controlled-v1
       - math-reasoning-controlled-v2
       - math-reasoning-controlled-v3
+      - gsm8k-main-test
     required_capabilities:
       text_input: true
     prompt:
@@ -126,3 +128,24 @@ tasks:
         Solve the problem and return only the final answer in the requested structured
         field. Do not include private chain-of-thought or intermediate reasoning.
       output_field: answer
+  
+  json-schema-compliance:
+    version: "1"
+    plugin: json-schema-compliance
+    evaluator: json-schema-validation
+    evaluator_version: "1"
+    compatible_datasets:
+      - jsonschemabench-github-easy-test
+    required_capabilities:
+      text_input: true
+    prompt:
+      id: json-schema-compliance-v1
+      version: "1"
+    metrics:
+      - name: schema_valid_rate
+        primary: true
+      - name: parse_valid
+    options:
+      instruction: >-
+        Generate an example JSON object satisfying the provided JSON Schema.
+        Return only the JSON object and do not include explanatory text.

From d57f9668efd4949695b7eb96abac5b031e53ea23 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:45:26 +0200
Subject: [PATCH 09/33] feat(mcb): add bounded public smoke/core/extended
 budgets

---
 experiments/model-capability-benchmark/profiles.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/experiments/model-capability-benchmark/profiles.yaml b/experiments/model-capability-benchmark/profiles.yaml
index 5eae88ce..e8c2f94e 100644
--- a/experiments/model-capability-benchmark/profiles.yaml
+++ b/experiments/model-capability-benchmark/profiles.yaml
@@ -3,6 +3,9 @@ profiles:
     max_cases: 20
     datasets:
       banking77: 77
+      gsm8k-main-test: 12
+      squad-v2-validation: 12
+      jsonschemabench-github-easy-test: 12
       clinc150-oos: 50
       structured-output-controlled-v1: null
       qa-abstention-controlled-v1: null
@@ -16,6 +19,9 @@ profiles:
     max_cases: 100
     datasets:
       banking77: 154
+      gsm8k-main-test: 40
+      squad-v2-validation: 60
+      jsonschemabench-github-easy-test: 40
       clinc150-oos: 100
       structured-output-controlled-v2: 60
       qa-abstention-controlled-v2: 60
@@ -26,6 +32,9 @@ profiles:
     max_cases: 500
     datasets:
       banking77: 770
+      gsm8k-main-test: 200
+      squad-v2-validation: 200
+      jsonschemabench-github-easy-test: 150
       clinc150-oos: 500
       structured-output-controlled-v2: 60
       qa-abstention-controlled-v2: 60

From a98a899464f7d2d1ec3cfbcd1b75a8b58c97e485 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:45:29 +0200
Subject: [PATCH 10/33] build(mcb): install Parquet reader for public datasets

---
 experiments/model-capability-benchmark/pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/experiments/model-capability-benchmark/pyproject.toml b/experiments/model-capability-benchmark/pyproject.toml
index 45b88cd9..9af27be4 100644
--- a/experiments/model-capability-benchmark/pyproject.toml
+++ b/experiments/model-capability-benchmark/pyproject.toml
@@ -8,6 +8,7 @@ dependencies = [
   "jsonschema>=4.23",
   "openai>=1.0",
   "pyyaml>=6.0.3",
+  "pyarrow>=17.0",
   "duckdb>=1.2.0",
   "typesafe-sdk>=0.7.2",
   "torch>=2.14.1",

From 790737f291589fb0f31ce61278bd4e56854b46e8 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:45:46 +0200
Subject: [PATCH 11/33] feat(mcb): support SQuAD 2.0 gold aliases and official
 text normalization

---
 .../tasks/qa_abstention.py                    | 26 +++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py
index 732aeb7a..8f5f16c1 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/qa_abstention.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import re
+import string
 from collections.abc import Mapping
 from typing import Any
 
@@ -49,6 +51,27 @@ def _expected(sample: Sample) -> tuple[str, bool]:
     def _normalize_answer(value: str) -> str:
         return " ".join(value.strip().lower().split())
 
+    @staticmethod
+    def _answer_matches(sample: Sample, prediction: str, expected: str) -> bool:
+        """Preserve legacy exact semantics; SQuAD uses its published normalization."""
+        if sample.metadata.get("answer_normalization") != "squad-v2":
+            return QaAbstentionTask._normalize_answer(prediction) == (
+                QaAbstentionTask._normalize_answer(expected)
+            )
+        aliases = sample.metadata.get("accepted_answers")
+        if not isinstance(aliases, (list, tuple)) or not aliases:
+            aliases = [expected]
+
+        def squad_normalize(value: str) -> str:
+            lowered = value.lower()
+            no_punctuation = "".join(c for c in lowered if c not in string.punctuation)
+            no_articles = re.sub(r"\b(a|an|the)\b", " ", no_punctuation)
+            return " ".join(no_articles.split())
+
+        return squad_normalize(prediction) in {
+            squad_normalize(alias) for alias in aliases if isinstance(alias, str)
+        }
+
     def build_request(
         self,
         sample: Sample,
@@ -106,8 +129,7 @@ def evaluate(
         exact_match: float | None
         if answerable and valid and not abstain and predicted_answer is not None:
             exact_match = float(
-                self._normalize_answer(predicted_answer)
-                == self._normalize_answer(expected_answer)
+                self._answer_matches(sample, predicted_answer, expected_answer)
             )
         elif answerable:
             exact_match = 0.0

From 6fa6c546453cf9357086f8926ecd8228c6fd127e Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:46:16 +0200
Subject: [PATCH 12/33] feat(mcb): add opt-in public benchmark suite preserving
 canonical core

---
 .../suite_public.yaml                         | 140 ++++++++++++++++++
 1 file changed, 140 insertions(+)
 create mode 100644 experiments/model-capability-benchmark/suite_public.yaml

diff --git a/experiments/model-capability-benchmark/suite_public.yaml b/experiments/model-capability-benchmark/suite_public.yaml
new file mode 100644
index 00000000..355ff7bd
--- /dev/null
+++ b/experiments/model-capability-benchmark/suite_public.yaml
@@ -0,0 +1,140 @@
+suite:
+  id: capability-public-benchmarks
+  version: "1"
+  description: >-
+    Opt-in pinned public test/validation subsets. These MCB-specific protocols
+    are not official full-benchmark leaderboard scores.
+  default_profile: smoke
+  generation:
+    temperature: 0.0
+    max_output_tokens: 2048
+    seed: 42
+
+  capabilities:
+    public-mathematical-reasoning:
+      task: mathematical-reasoning
+      datasets: [gsm8k-main-test]
+      description: GSM8K main/test numeric final-answer accuracy, MCB 0-shot JSON protocol.
+      tags: [math, gsm8k, public, subset]
+      benchmark:
+        smoke:
+          max_cases: 12
+          target_local_seconds: 180
+          hard_local_seconds: 300
+        core:
+          max_cases: 40
+          target_local_seconds: 600
+          hard_local_seconds: 900
+        extended:
+          max_cases: 200
+          hard_local_seconds: 3600
+      comparison:
+        practical_delta: 0.05
+        metric: final_answer_accuracy
+      metrics:
+        - name: final_answer_accuracy
+          source: task_metric
+          reducer: mean
+          primary: true
+        - name: parse_valid
+          source: task_metric
+          reducer: mean
+        - name: latency_p95_ms
+          source: inference
+          reducer: p95
+          field: latency_ms
+
+    public-qa-abstention:
+      task: qa-abstention
+      datasets: [squad-v2-validation]
+      description: SQuAD 2.0 validation; MCB JSON answer/abstain protocol, alias-aware exact match.
+      tags: [qa, abstention, squad-v2, public, subset]
+      benchmark:
+        smoke:
+          max_cases: 12
+          selection:
+            strategy: stratified
+            quotas:
+              - match: {answerable: true}
+                count: 6
+              - match: {answerable: false}
+                count: 6
+          target_local_seconds: 180
+          hard_local_seconds: 300
+        core:
+          max_cases: 60
+          selection:
+            strategy: stratified
+            quotas:
+              - match: {answerable: true}
+                count: 30
+              - match: {answerable: false}
+                count: 30
+          target_local_seconds: 600
+          hard_local_seconds: 900
+        extended:
+          max_cases: 200
+          selection:
+            strategy: stratified
+            quotas:
+              - match: {answerable: true}
+                count: 100
+              - match: {answerable: false}
+                count: 100
+          hard_local_seconds: 3600
+      comparison:
+        practical_delta: 0.05
+        metric: qa_correct
+      metrics:
+        - name: qa_correct
+          source: task_metric
+          reducer: mean
+          primary: true
+        - name: answerability_accuracy
+          source: task_metric
+          reducer: mean
+        - name: exact_match
+          source: task_metric
+          reducer: mean
+        - name: parse_valid
+          source: task_metric
+          reducer: mean
+        - name: latency_p95_ms
+          source: inference
+          reducer: p95
+          field: latency_ms
+
+    public-json-schema-compliance:
+      task: json-schema-compliance
+      datasets: [jsonschemabench-github-easy-test]
+      description: >-
+        JSONSchemaBench Github_easy/test; object-root, locally-resolvable, bounded
+        schemas only. Compliance scoring, not semantic extraction or official harness.
+      tags: [structured-output, json-schema, public, subset]
+      benchmark:
+        smoke:
+          max_cases: 12
+          target_local_seconds: 180
+          hard_local_seconds: 300
+        core:
+          max_cases: 40
+          target_local_seconds: 600
+          hard_local_seconds: 900
+        extended:
+          max_cases: 150
+          hard_local_seconds: 3600
+      comparison:
+        practical_delta: 0.05
+        metric: schema_valid_rate
+      metrics:
+        - name: schema_valid_rate
+          source: task_metric
+          reducer: mean
+          primary: true
+        - name: parse_valid
+          source: task_metric
+          reducer: mean
+        - name: latency_p95_ms
+          source: inference
+          reducer: p95
+          field: latency_ms

From fc7f7cf4e325638850b516d62f3e647248ca98e3 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:46:29 +0200
Subject: [PATCH 13/33] feat(mcb): select opt-in suite file without mutating
 default suite

---
 .../src/model_capability_bench/suite.py                   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
index 621c7b66..adc23727 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
@@ -99,13 +99,17 @@ def plan_matrix(
         return tuple(arms)
 
 
-def load_capability_suite(root: str | Path) -> CapabilitySuiteBundle:
+def load_capability_suite(
+    root: str | Path, *, suite_filename: str = "suite.yaml"
+) -> CapabilitySuiteBundle:
+    if Path(suite_filename).name != suite_filename or not suite_filename.endswith(".yaml"):
+        raise ValueError("suite_filename must be a YAML basename within the benchmark root")
     root_path = Path(root)
     models = load_registry(root_path / "models.yaml")
     tasks = build_task_registry(root_path / "tasks.yaml")
     datasets = build_dataset_registry(root_path / "datasets.yaml")
     profiles = load_dataset_profiles(root_path / "profiles.yaml")
-    suite = load_suite_spec(root_path / "suite.yaml")
+    suite = load_suite_spec(root_path / suite_filename)
     resolved = validate_suite(
         suite,
         tasks=tasks,

From ed7c3c44c46e5d8152a6c2471f4050777844cd61 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:46:32 +0200
Subject: [PATCH 14/33] feat(mcb): CLI suite switch for public benchmark
 campaign

---
 .../src/model_capability_bench/cli.py                     | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/cli.py b/experiments/model-capability-benchmark/src/model_capability_bench/cli.py
index b27eb82b..74777e99 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/cli.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/cli.py
@@ -71,6 +71,10 @@ def build_parser() -> argparse.ArgumentParser:
         description="Unified model capability benchmark runner.",
     )
     parser.add_argument("--root", type=Path, default=ROOT)
+    parser.add_argument(
+        "--suite", default="suite.yaml",
+        help="Versioned suite YAML basename (e.g. suite_public.yaml); place before command.",
+    )
     sub = parser.add_subparsers(dest="command", required=True)
 
     run = sub.add_parser("run", help="Execute a model × capability benchmark matrix.")
@@ -284,7 +288,7 @@ def main() -> int:
         output = args.output_dir or root / "results" / "runs" / args.run_id
         summary = consolidate_runs(
             sources=args.run_dir, output_dir=output, run_id=args.run_id,
-            run_group=args.run_group, suite=load_capability_suite(root), plan_only=args.plan_only,
+            run_group=args.run_group, suite=load_capability_suite(root, suite_filename=args.suite), plan_only=args.plan_only,
         )
         if not args.plan_only:
             reporting_config = load_reporting_config(root)
@@ -448,7 +452,7 @@ def main() -> int:
         )
         return 0
 
-    bundle = load_capability_suite(root)
+    bundle = load_capability_suite(root, suite_filename=args.suite)
     defaults = load_runner_defaults(root)
 
     if args.command in {"models", "tasks", "datasets", "capabilities"}:

From 5f6542f08869ffe827effdff79a48cd46238cad9 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:47:28 +0200
Subject: [PATCH 15/33] fix(mcb): retain active suite filename in bundle
 provenance

---
 .../src/model_capability_bench/suite.py                         | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
index adc23727..728074c7 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
@@ -50,6 +50,7 @@ class CapabilitySuiteBundle:
     tasks: TaskRegistry
     datasets: DatasetRegistry
     profiles: Mapping[str, DatasetProfileSpec]
+    suite_filename: str = "suite.yaml"
 
     def plan_matrix(
         self,
@@ -115,6 +116,7 @@ def load_capability_suite(
         tasks=tasks,
         datasets=datasets,
         profiles=profiles,
+        suite_filename=suite_filename,
     )
     return CapabilitySuiteBundle(
         root=root_path,

From ae0f5a522759ea8be7a4f69ec9a11b51d9f60b8b Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:47:31 +0200
Subject: [PATCH 16/33] fix(mcb): checksum and persist selected public suite in
 run manifest

---
 .../src/model_capability_bench/runner/manifest.py        | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py b/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py
index aebbc400..a1405033 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/runner/manifest.py
@@ -30,12 +30,14 @@
 )
 
 
-def _config_checksums(root: Path) -> dict[str, str]:
+def _config_checksums(root: Path, *, suite_filename: str = "suite.yaml") -> dict[str, str]:
     checksums: dict[str, str] = {}
     for name in CONFIG_FILES:
         path = root / name
         if path.is_file():
             checksums[name] = sha256_file(path)
+    if suite_filename not in checksums:
+        checksums[suite_filename] = sha256_file(root / suite_filename)
     return checksums
 
 
@@ -182,6 +184,7 @@ def write_run_artifacts(
         "suite": {
             "suite_id": suite.suite.suite_id,
             "version": suite.suite.version,
+            "suite_file": suite.suite_filename,
             "profile": config.profile,
             "seed": config.seed,
         },
@@ -200,7 +203,9 @@ def write_run_artifacts(
         },
         "capabilities": selected_capabilities,
         "signatures": signatures,
-        "config_checksums": _config_checksums(suite.root),
+        "config_checksums": _config_checksums(
+            suite.root, suite_filename=suite.suite_filename
+        ),
         "evidence": {
             "state": "state.jsonl",
             "raw": "raw.jsonl",

From 43ae1a588fc6dc1f274e503f9858a765d21c509f Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:47:58 +0200
Subject: [PATCH 17/33] test(mcb): update registry contract for public
 benchmark adapters

---
 .../tests/test_dataset_adapters.py                          | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/experiments/model-capability-benchmark/tests/test_dataset_adapters.py b/experiments/model-capability-benchmark/tests/test_dataset_adapters.py
index f11e5e38..dd41b028 100644
--- a/experiments/model-capability-benchmark/tests/test_dataset_adapters.py
+++ b/experiments/model-capability-benchmark/tests/test_dataset_adapters.py
@@ -69,15 +69,18 @@ def test_catalog_profiles_and_task_references_are_consistent() -> None:
     profiles = load_dataset_profiles(PROFILES)
 
     assert datasets.summary() == {
-        "datasets": 9,
+        "datasets": 12,
         "dataset_ids": [
             "banking77",
             "clinc150-oos",
+            "gsm8k-main-test",
+            "jsonschemabench-github-easy-test",
             "math-reasoning-controlled-v1",
             "math-reasoning-controlled-v2",
             "math-reasoning-controlled-v3",
             "qa-abstention-controlled-v1",
             "qa-abstention-controlled-v2",
+            "squad-v2-validation",
             "structured-output-controlled-v1",
             "structured-output-controlled-v2",
         ],
@@ -85,6 +88,7 @@ def test_catalog_profiles_and_task_references_are_consistent() -> None:
             "banking77",
             "clinc150-oos",
             "controlled-yaml",
+            "public-parquet",
             "structured-output-controlled",
         ],
     }

From c1e3799d6d34cbe18029af598bb9b8a862189d2d Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:48:42 +0200
Subject: [PATCH 18/33] fix(mcb): read pinned Parquet with existing DuckDB
 dependency

---
 .../datasets/public_benchmarks.py                | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
index 94c2e61d..d1bd5e25 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
@@ -26,13 +26,15 @@
 
 
 def _rows(path, columns: tuple[str, ...]) -> list[dict]:
-    try:
-        import pyarrow.parquet as pq
-    except ImportError as exc:
-        raise RuntimeError(
-            "Public Parquet datasets require pyarrow; install MCB dependencies"
-        ) from exc
-    return pq.read_table(path, columns=list(columns)).to_pylist()
+    import duckdb
+
+    selected = ", ".join(f'"{column}"' for column in columns)
+    with duckdb.connect(":memory:") as conn:
+        result = conn.execute(
+            f"SELECT {selected} FROM read_parquet(?)", [str(path)]
+        )
+        names = [item[0] for item in result.description]
+        return [dict(zip(names, row, strict=True)) for row in result.fetchall()]
 
 
 def _required_text(value: object, *, case: str, field: str) -> str:

From 876ea523ac6811ef7743496e431057098b1f75e1 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:48:44 +0200
Subject: [PATCH 19/33] build(mcb): reuse existing DuckDB; avoid lockfile churn

---
 experiments/model-capability-benchmark/pyproject.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/experiments/model-capability-benchmark/pyproject.toml b/experiments/model-capability-benchmark/pyproject.toml
index 9af27be4..45b88cd9 100644
--- a/experiments/model-capability-benchmark/pyproject.toml
+++ b/experiments/model-capability-benchmark/pyproject.toml
@@ -8,7 +8,6 @@ dependencies = [
   "jsonschema>=4.23",
   "openai>=1.0",
   "pyyaml>=6.0.3",
-  "pyarrow>=17.0",
   "duckdb>=1.2.0",
   "typesafe-sdk>=0.7.2",
   "torch>=2.14.1",

From 6eb5382ff0fc0034b99e4d249d5146e1fed68e63 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:49:42 +0200
Subject: [PATCH 20/33] test(mcb): cover pinned public adapters, SQuAD alias
 scoring, selection, schema compliance

---
 .../tests/test_public_benchmarks.py           | 189 ++++++++++++++++++
 1 file changed, 189 insertions(+)
 create mode 100644 experiments/model-capability-benchmark/tests/test_public_benchmarks.py

diff --git a/experiments/model-capability-benchmark/tests/test_public_benchmarks.py b/experiments/model-capability-benchmark/tests/test_public_benchmarks.py
new file mode 100644
index 00000000..6c8643d2
--- /dev/null
+++ b/experiments/model-capability-benchmark/tests/test_public_benchmarks.py
@@ -0,0 +1,189 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+
+from benchmark_core import (
+    DatasetLoadContext,
+    InferenceResult,
+    TaskExecutionContext,
+)
+
+from model_capability_bench import build_dataset_registry, load_capability_suite
+from model_capability_bench.datasets import public_benchmarks
+from model_capability_bench.runner.capability_data import load_capability_datasets
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _fixture_spec(dataset_id: str):
+    return build_dataset_registry(ROOT / "datasets.yaml").get(dataset_id).spec
+
+
+def test_opt_in_suite_does_not_change_legacy_core() -> None:
+    old = load_capability_suite(ROOT)
+    assert old.suite.suite_id == "capability-core"
+    assert len(old.resolved_capabilities) == 5
+
+    public = load_capability_suite(ROOT, suite_filename="suite_public.yaml")
+    assert public.suite.suite_id == "capability-public-benchmarks"
+    assert len(public.resolved_capabilities) == 3
+    assert public.suite_filename == "suite_public.yaml"
+    assert {c.spec.capability_id for c in public.resolved_capabilities} == {
+        "public-mathematical-reasoning",
+        "public-qa-abstention",
+        "public-json-schema-compliance",
+    }
+
+
+def test_gsm8k_gold_final_answers_are_numeric_and_source_rows_preserved() -> None:
+    adapter = public_benchmarks.PublicParquetBenchmark(_fixture_spec("gsm8k-main-test"))
+    rows = [
+        {"question": "If a pack costs $1, how much do 1,200 packs cost?",
+         "answer": "Compute the total.\n#### 1,200"},
+        {"question": "Find the remaining balance.",
+         "answer": "Subtract amounts.\n#### 3.50"},
+    ]
+    samples, stats = adapter._convert(rows)
+    assert [case.expected for case in samples] == ["1200", "3.5"]
+    assert [case.metadata["source_index"] for case in samples] == [0, 1]
+    assert len(set(case.sample_id for case in samples)) == 2
+    assert stats == {"source_rows": 2, "excluded_rows": 0}
+
+
+def test_squad_gold_aliases_and_abstention_preserved() -> None:
+    adapter = public_benchmarks.PublicParquetBenchmark(
+        _fixture_spec("squad-v2-validation")
+    )
+    rows = [
+        {"id": "abc1", "title": "Example", "context": "The answer is the 10th century.",
+         "question": "When?", "answers": {
+             "text": ["the 10th century", "10th century"],
+             "answer_start": [14, 18],
+         }},
+        {"id": "abc2", "title": "Example", "context": "It does not say.",
+         "question": "When?", "answers": {"text": [], "answer_start": []}},
+    ]
+    samples, _ = adapter._convert(rows)
+    assert samples[0].metadata["accepted_answers"] == [
+        "the 10th century", "10th century"
+    ]
+    assert samples[1].expected == {"answer": "", "answerable": False}
+
+    suite = load_capability_suite(ROOT, suite_filename="suite_public.yaml")
+    task = suite.tasks.get("qa-abstention")
+    context = TaskExecutionContext(
+        run_id="public-test", dataset_id="squad-v2-validation",
+        profile="core", generation=suite.suite.generation,
+    )
+    correct = task.evaluate(
+        samples[0],
+        InferenceResult(
+            provider_id="test", model_id="test",
+            raw_output={"answer": "10th century", "abstain": False},
+            normalized_output={"answer": "10th century", "abstain": False},
+            latency_ms=0.0,
+        ),
+        context,
+    )
+    assert next(m.value for m in correct.metrics if m.name == "qa_correct") == 1.0
+
+    abstained = task.evaluate(
+        samples[1],
+        InferenceResult(
+            provider_id="test", model_id="test",
+            raw_output={"answer": "", "abstain": True},
+            normalized_output={"answer": "", "abstain": True},
+            latency_ms=0.0,
+        ),
+        context,
+    )
+    assert next(m.value for m in abstained.metrics if m.name == "qa_correct") == 1.0
+
+
+def test_squad_core_selects_balanced_answerability_without_network(
+    monkeypatch, tmp_path: Path
+) -> None:
+    rows = [
+        {"id": f"{n:05d}", "title": "Fixture",
+         "context": f"Person {n} lives in Rome.",
+         "question": f"Where does person {n} live?",
+         "answers": {
+             "text": ["Rome"] if n % 2 else [],
+             "answer_start": [18] if n % 2 else [],
+         }}
+        for n in range(160)
+    ]
+    path = tmp_path / "fixture.parquet"
+    path.write_bytes(b"test-only data")
+    monkeypatch.setattr(public_benchmarks, "single_cached_source", lambda *_: path)
+    monkeypatch.setattr(public_benchmarks, "_rows", lambda *_: rows)
+    bundle = load_capability_suite(ROOT, suite_filename="suite_public.yaml")
+    capability = next(
+        c for c in bundle.resolved_capabilities
+        if c.spec.capability_id == "public-qa-abstention"
+    )
+    kwargs = dict(
+        suite=bundle, capability=capability,
+        profile=bundle.profiles["core"], profile_id="core",
+        cache_dir=tmp_path, seed=42,
+    )
+    a = load_capability_datasets(**kwargs, cache={})["squad-v2-validation"]
+    b = load_capability_datasets(**kwargs, cache={})["squad-v2-validation"]
+    assert len(a.samples) == 60
+    assert a.selection_fingerprint == b.selection_fingerprint
+    assert Counter(c.metadata["answerable"] for c in a.samples) == {
+        True: 30, False: 30
+    }
+    assert set(a.source_checksums) == {"source"}
+
+
+def test_jsonschemabench_only_scores_schema_compliance() -> None:
+    adapter = public_benchmarks.PublicParquetBenchmark(
+        _fixture_spec("jsonschemabench-github-easy-test")
+    )
+    schema = {
+        "type": "object",
+        "properties": {"count": {"type": "integer", "minimum": 2}},
+        "required": ["count"], "additionalProperties": False,
+    }
+    remote = {"type": "object", "properties": {
+        "external": {"$ref": "https://example.org/anything"}
+    }}
+    cases, stats = adapter._convert([
+        {"unique_id": "ok", "json_schema": json.dumps(schema)},
+        {"unique_id": "remote", "json_schema": json.dumps(remote)},
+    ])
+    assert len(cases) == 1
+    assert stats["excluded_rows"] == 1
+    assert cases[0].expected is None
+
+    suite = load_capability_suite(ROOT, suite_filename="suite_public.yaml")
+    task = suite.tasks.get("json-schema-compliance")
+    ctx = TaskExecutionContext(
+        run_id="public-test", dataset_id="jsonschemabench-github-easy-test",
+        profile="core", generation=suite.suite.generation,
+    )
+
+    def score(value):
+        result = task.evaluate(
+            cases[0],
+            InferenceResult(provider_id="test", model_id="test",
+                            raw_output=value, normalized_output=value,
+                            latency_ms=0.0),
+            ctx,
+        )
+        return next(m.value for m in result.metrics
+                    if m.name == "schema_valid_rate")
+
+    assert score({"count": 3}) == 1.0
+    assert score({"count": 1}) == 0.0
+    assert score({"count": "3"}) == 0.0
+
+
+def test_public_suite_path_is_restricted() -> None:
+    import pytest
+
+    with pytest.raises(ValueError, match="YAML basename"):
+        load_capability_suite(ROOT, suite_filename="../other.yaml")

From 2be89c432be75affb35e1abacda644d7ba6c5eb9 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:50:01 +0200
Subject: [PATCH 21/33] fix(mcb): resolve all capabilities only within selected
 suite

---
 .../src/model_capability_bench/cli.py                         | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/cli.py b/experiments/model-capability-benchmark/src/model_capability_bench/cli.py
index 74777e99..9325fa6f 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/cli.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/cli.py
@@ -60,7 +60,9 @@ def _selection(value: str, available: list[str]) -> tuple[str, ...]:
 
 def _select_capabilities(bundle, raw_value: str) -> tuple[str, ...]:
     canonical = [c.spec.capability_id for c in bundle.resolved_capabilities]
-    available = canonical + list(CAPABILITY_ALIASES.keys())
+    available = canonical + [
+        alias for alias, target in CAPABILITY_ALIASES.items() if target in canonical
+    ]
     selected = _selection(raw_value, available)
     return tuple(dict.fromkeys(resolve_capability_id(c) for c in selected))
 

From a3d6f1ee725e0f804954938bec17d49c32654ebc Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:51:09 +0200
Subject: [PATCH 22/33] docs(mcb): document opt-in public benchmark protocol
 and reproducible commands

---
 .../PUBLIC_BENCHMARKS.md                      | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 experiments/model-capability-benchmark/PUBLIC_BENCHMARKS.md

diff --git a/experiments/model-capability-benchmark/PUBLIC_BENCHMARKS.md b/experiments/model-capability-benchmark/PUBLIC_BENCHMARKS.md
new file mode 100644
index 00000000..be19fa1b
--- /dev/null
+++ b/experiments/model-capability-benchmark/PUBLIC_BENCHMARKS.md
@@ -0,0 +1,77 @@
+# MCB public benchmark suite (opt-in)
+
+MCB has two independent suites. **The historical default stays unchanged**:
+
+- `suite.yaml` / `capability-core` v3: 468 controlled/public-label cases, including all previous benchmark revisions.
+- `suite_public.yaml` / `capability-public-benchmarks` v1: 140 small, reproducible public benchmark test/validation cases in Core.
+
+Public suite is an **MCB protocol on pinned public data**, **not** the official full dataset leaderboard protocol.
+We do not publish a 40-case GSM8K score as an official GSM8K full-test score.
+
+## Sources and provenance
+
+| MCB dataset ID | Upstream | Upstream split | License | Core |
+| --- | --- | --- | --- | ---: |
+| `gsm8k-main-test` | [openai/gsm8k](https://huggingface.co/datasets/openai/gsm8k) | `main/test` (1,319 total) | MIT | 40 |
+| `squad-v2-validation` | [rajpurkar/squad_v2](https://huggingface.co/datasets/rajpurkar/squad_v2) | `validation` (11,873 total) | CC BY-SA 4.0 | 60 (30 answerable + 30 unanswerable) |
+| `jsonschemabench-github-easy-test` | [epfl-dlab/JSONSchemaBench](https://huggingface.co/datasets/epfl-dlab/JSONSchemaBench) | `Github_easy/test` (577 upstream) | MIT | 40 |
+
+Each source URL in `datasets.yaml` pins a **full immutable upstream commit SHA**, the adapter uses the original Parquet file, and the cached source's SHA-256 is preserved in `source_checksums`. Dataset IDs, revisions, selected source indices, source splits, profiles and selection fingerprints persist in evidence. Case IDs never depend on random order. This requires no new dependency: MCB reads Parquet with its existing DuckDB dependency.
+
+The upstream dataset may have known contamination in pretrained models. Public provenance alone does **not** prove model training independence.
+
+**SQuAD protocol:** MCB asks for JSON `{answer, abstain}`, uses the official SQuAD text normalization (lowercase, remove punctuation, articles and extra whitespace), and accepts every upstream gold alias. It scores `qa_correct`, which requires a correct answer on answerable items and a correct abstention on impossible items. It does not compute the official SQuAD v2 threshold-tuned F1/EM leaderboard metric.
+
+**GSM8K protocol:** MCB extracts the upstream final numeric gold after `####`, normalizes decimals/commas and evaluates the final answer. The original worked solution is not leaked to the prompt. This is an MCB zero-shot JSON-output evaluation, not necessarily prompt-parity with an external published number.
+
+**JSONSchemaBench protocol:** MCB tests JSON Schema compliance, **not** semantic extraction or JSONSchemaBench's official constrained-decoding engine timings/coverage. Its v1 adapter deliberately selects only object-root schemas that satisfy local validator checks, have locally resolvable references and at most 12,000 schema characters. Rejected upstream rows are counted (`source_rows`, `excluded_rows`, `eligible_rows`) to make this reduced subset explicit. Unsupported remote references are never automatically fetched. The upstream dataset has no gold data values: `expected=None` is correct.
+
+## Commands
+
+From `experiments/model-capability-benchmark`:
+
+```bash
+uv sync --frozen --extra dev
+
+# Read configuration before downloading anything
+uv run --frozen model-bench --suite suite_public.yaml capabilities
+uv run --frozen model-bench --suite suite_public.yaml plan --profile core --capabilities all
+
+# Real preflight and cost/latency pilot on one selected model
+export MODEL=qwen3.5-2b-q4km
+uv run --frozen model-bench --suite suite_public.yaml validate-config --models "$MODEL"
+uv run --frozen model-bench --suite suite_public.yaml estimate \
+  --models "$MODEL" --profile core --capabilities all --pilot-cases 3
+
+# Distinct run group: never overwrite legacy results
+uv run --frozen model-bench --suite suite_public.yaml run \
+  --run-group public-benchmarks-v1 \
+  --models "$MODEL" --capabilities all --profile core
+
+# Offline projection and dashboard export from immutable run evidence
+uv run --frozen model-bench project --rebuild --export-dashboard
+```
+
+Existing default commands without `--suite` still run historical `suite.yaml`. Use the same
+public suite filename, seed, profile, commit, source artifact and evaluator versions for every compared model.
+
+Case budgets: `smoke = 12+12+12 = 36`, `core = 40+60+40 = 140`,
+`extended = 200+200+150 = 550`. Source files are cached and downloaded only on first
+load. Estimated run time is model-dependent. `plan` does not contact models or download data;
+`estimate` and `run` do.
+
+## Science and interpretation guards
+
+- MCB stores **raw sources and actual selected case fingerprints**, so any same-case paired comparison remains reproducible.
+- Do not compare public v1 scores directly to controlled v2/v3 scores, or call them equivalent despite both measuring math or QA.
+- Do not give official benchmark leaderboard labels or cite official test totals as run sample counts.
+- Report paired `n`, confidence intervals, invalid-output rates and missing resource data separately.
+- SQuAD and JSONSchemaBench use different concepts of correctness; aggregating into one undifferentiated score is not recommended.
+- JSONSchemaBench's object/root-size/reference filtering must remain versioned and visible, because it affects measured difficulty.
+- Before external publication, perform a real-provider smoke + canonical run and review a sample of gold normalization and schema evaluator outputs.
+
+## Recommended follow-on
+
+Add a second JSONSchemaBench subset (`Github_medium`) only after the object-root compliance path passes
+real provider validation. Add structured-output **semantic extraction** from benchmark sources separately;
+schema compliance alone cannot replace the original controlled extraction suite.

From dd992189d259764d1522b0a5c97e5a970b833ffb Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:51:31 +0200
Subject: [PATCH 23/33] docs(mcb): link the new public benchmark suite from
 README

---
 experiments/model-capability-benchmark/README.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/experiments/model-capability-benchmark/README.md b/experiments/model-capability-benchmark/README.md
index e90a2e43..8441ecc4 100644
--- a/experiments/model-capability-benchmark/README.md
+++ b/experiments/model-capability-benchmark/README.md
@@ -103,6 +103,22 @@ read from the environment and is never written to the registry or manifest.
 
 See `.env.example` for the currently supported environment variables.
 
+## Public benchmark suite (opt-in)
+
+A second, independently versioned suite (`suite_public.yaml`) integrates pinned upstream
+test/validation subsets of **GSM8K**, **SQuAD 2.0** and **JSONSchemaBench**.
+It has 36 smoke / 140 core / 550 extended cases and does not replace the canonical
+468-case `suite.yaml`. The UI and persisted evidence keep the benchmark lineages distinct.
+JSONSchemaBench measures schema compliance, not semantic field extraction.
+
+See **[PUBLIC_BENCHMARKS.md](PUBLIC_BENCHMARKS.md)** for sources, data licenses,
+normalization/scoring semantics, reproducible run commands and the explicit
+differences from official full-benchmark leaderboard protocols.
+
+```bash
+uv run --frozen model-bench --suite suite_public.yaml plan --profile core --capabilities all
+```
+
 ## Capability suite
 
 `suite.yaml` now composes five capability families: BANKING77 intent classification,

From 8ef57d9b4c5e23294622df643529938c93221634 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:53:15 +0200
Subject: [PATCH 24/33] fix(mcb): satisfy strict lint in public adapter

---
 .../datasets/public_benchmarks.py                  | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
index d1bd5e25..81544c40 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/datasets/public_benchmarks.py
@@ -19,6 +19,7 @@
     fingerprint_values,
     sha256_file,
 )
+from jsonschema.exceptions import SchemaError
 from jsonschema.validators import validator_for
 
 from model_capability_bench.datasets.common import single_cached_source
@@ -59,7 +60,7 @@ def _gsm_final_answer(solution: str) -> str:
 
 def _squad_answers(value: object) -> tuple[str, ...]:
     if not isinstance(value, Mapping):
-        raise ValueError("SQuAD answers must be an object")
+        raise TypeError("SQuAD answers must be an object")
     texts = value.get("text")
     if not isinstance(texts, list) or not all(isinstance(t, str) for t in texts):
         raise ValueError("SQuAD answers.text must be a string list")
@@ -69,10 +70,11 @@ def _squad_answers(value: object) -> tuple[str, ...]:
 def _unsafe_ref(value: object) -> bool:
     if isinstance(value, Mapping):
         for key, item in value.items():
-            if key in {"$ref", "$dynamicRef", "$recursiveRef"}:
-                # Avoid implicit network access and runtime-dependent reference resolution.
-                if not isinstance(item, str) or not item.startswith("#/"):
-                    return True
+            # Never fetch remote references implicitly while scoring.
+            if key in {"$ref", "$dynamicRef", "$recursiveRef"} and (
+                not isinstance(item, str) or not item.startswith("#/")
+            ):
+                return True
             if _unsafe_ref(item):
                 return True
     elif isinstance(value, list):
@@ -91,7 +93,7 @@ def _schema(value: object, *, max_chars: int) -> dict | None:
             return None
         validator_for(schema).check_schema(schema)
         return schema
-    except Exception:
+    except (ValueError, TypeError, SchemaError, RecursionError):
         # Third-party schema catalogs can include dialects/extensions outside
         # the local JSON Schema validator. Those cases are explicitly excluded.
         return None

From 612e6b4d1638acc72c6be322e131ad0edbd77d17 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:53:18 +0200
Subject: [PATCH 25/33] fix(mcb): prefer TypeError for invalid schema type

---
 .../src/model_capability_bench/tasks/schema_compliance.py       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py
index 4fe0994b..afb82343 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/schema_compliance.py
@@ -32,7 +32,7 @@ def spec(self) -> TaskSpec:
     def _schema(self, sample: Sample) -> Mapping:
         schema = sample.metadata.get("response_schema")
         if not isinstance(schema, Mapping):
-            raise ValueError(f"{sample.sample_id}: missing response_schema")
+            raise TypeError(f"{sample.sample_id}: missing response_schema")
         validator_for(schema).check_schema(schema)
         return schema
 

From 6a26321f092b8a8eadf097327221fed71a16c32f Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:53:20 +0200
Subject: [PATCH 26/33] style(mcb): sort task imports

---
 .../src/model_capability_bench/tasks/registry.py                | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py
index 4b82afde..da0b4e2b 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/registry.py
@@ -9,8 +9,8 @@
     IntentClassificationTask,
 )
 from model_capability_bench.tasks.qa_abstention import QaAbstentionTask
-from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask
 from model_capability_bench.tasks.reasoning import MathematicalReasoningTask
+from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask
 from model_capability_bench.tasks.structured_output import StructuredOutputTask
 
 

From f70147dc4251758c157c2ff2d9671238ba9739cb Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:53:23 +0200
Subject: [PATCH 27/33] style(mcb): sort task exports

---
 .../src/model_capability_bench/tasks/__init__.py              | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py
index 0a931c09..05717205 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/tasks/__init__.py
@@ -3,17 +3,17 @@
     IntentClassificationTask,
 )
 from model_capability_bench.tasks.qa_abstention import QaAbstentionTask
-from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask
 from model_capability_bench.tasks.reasoning import MathematicalReasoningTask
 from model_capability_bench.tasks.registry import build_task_plugins, build_task_registry
+from model_capability_bench.tasks.schema_compliance import JsonSchemaComplianceTask
 from model_capability_bench.tasks.structured_output import StructuredOutputTask
 
 __all__ = [
     "CalibratedIntentClassificationTask",
     "IntentClassificationTask",
+    "JsonSchemaComplianceTask",
     "MathematicalReasoningTask",
     "QaAbstentionTask",
-    "JsonSchemaComplianceTask",
     "StructuredOutputTask",
     "build_task_plugins",
     "build_task_registry",

From 0910b4f0be51369461c3416b3a9c66408cb5db70 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:53:26 +0200
Subject: [PATCH 28/33] style(mcb): satisfy strict Ruff test lint

---
 .../tests/test_public_benchmarks.py                 | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/experiments/model-capability-benchmark/tests/test_public_benchmarks.py b/experiments/model-capability-benchmark/tests/test_public_benchmarks.py
index 6c8643d2..51f760bf 100644
--- a/experiments/model-capability-benchmark/tests/test_public_benchmarks.py
+++ b/experiments/model-capability-benchmark/tests/test_public_benchmarks.py
@@ -5,7 +5,6 @@
 from pathlib import Path
 
 from benchmark_core import (
-    DatasetLoadContext,
     InferenceResult,
     TaskExecutionContext,
 )
@@ -48,7 +47,7 @@ def test_gsm8k_gold_final_answers_are_numeric_and_source_rows_preserved() -> Non
     samples, stats = adapter._convert(rows)
     assert [case.expected for case in samples] == ["1200", "3.5"]
     assert [case.metadata["source_index"] for case in samples] == [0, 1]
-    assert len(set(case.sample_id for case in samples)) == 2
+    assert len({case.sample_id for case in samples}) == 2
     assert stats == {"source_rows": 2, "excluded_rows": 0}
 
 
@@ -124,11 +123,11 @@ def test_squad_core_selects_balanced_answerability_without_network(
         c for c in bundle.resolved_capabilities
         if c.spec.capability_id == "public-qa-abstention"
     )
-    kwargs = dict(
-        suite=bundle, capability=capability,
-        profile=bundle.profiles["core"], profile_id="core",
-        cache_dir=tmp_path, seed=42,
-    )
+    kwargs = {
+        "suite": bundle, "capability": capability,
+        "profile": bundle.profiles["core"], "profile_id": "core",
+        "cache_dir": tmp_path, "seed": 42,
+    }
     a = load_capability_datasets(**kwargs, cache={})["squad-v2-validation"]
     b = load_capability_datasets(**kwargs, cache={})["squad-v2-validation"]
     assert len(a.samples) == 60

From 2e8f3b55e6b1f422830937649c15c7076852ba7f Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:54:36 +0200
Subject: [PATCH 29/33] fix(mcb): pass suite provenance only to bundle, not
 core validator

---
 .../src/model_capability_bench/suite.py                          | 1 -
 1 file changed, 1 deletion(-)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
index 728074c7..0486d2e4 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
@@ -116,7 +116,6 @@ def load_capability_suite(
         tasks=tasks,
         datasets=datasets,
         profiles=profiles,
-        suite_filename=suite_filename,
     )
     return CapabilitySuiteBundle(
         root=root_path,

From 6e05b00a84367a48076064a82d3781987ac9f57b Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:55:01 +0200
Subject: [PATCH 30/33] fix(mcb): persist active suite file in returned bundle

---
 .../src/model_capability_bench/suite.py                          | 1 +
 1 file changed, 1 insertion(+)

diff --git a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
index 0486d2e4..d29d8540 100644
--- a/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
+++ b/experiments/model-capability-benchmark/src/model_capability_bench/suite.py
@@ -125,4 +125,5 @@ def load_capability_suite(
         tasks=tasks,
         datasets=datasets,
         profiles=profiles,
+        suite_filename=suite_filename,
     )

From 26e1ac84e1369d9f7046c64bf6b27249a9f318f2 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:55:39 +0200
Subject: [PATCH 31/33] test(mcb): distinguish pinned HF dataset config main
 from moving branch main

---
 .../tests/test_dataset_adapters.py               | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/experiments/model-capability-benchmark/tests/test_dataset_adapters.py b/experiments/model-capability-benchmark/tests/test_dataset_adapters.py
index dd41b028..14590096 100644
--- a/experiments/model-capability-benchmark/tests/test_dataset_adapters.py
+++ b/experiments/model-capability-benchmark/tests/test_dataset_adapters.py
@@ -196,7 +196,6 @@ def test_profiles_are_configuration_driven() -> None:
 
 
 def test_public_dataset_catalog_uses_pinned_revisions_and_no_moving_main() -> None:
-    text = DATASETS.read_text(encoding="utf-8")
     registry = build_dataset_registry(DATASETS)
 
     assert registry.get("banking77").spec.revision == (
@@ -205,8 +204,19 @@ def test_public_dataset_catalog_uses_pinned_revisions_and_no_moving_main() -> No
     assert registry.get("clinc150-oos").spec.revision == (
         "48a0e1cff8f43dd4d0836ecb4ed5df08733e3d2e"
     )
-    assert "/main/" not in text
-    assert "refs/heads/main" not in text
+    for dataset in registry.select():
+        if dataset.spec.cache_mode != "download":
+            continue
+        files = dataset.spec.options.get("files") or {
+            "source": dataset.spec.options.get("file")
+        }
+        for entry in files.values():
+            url = entry["url"]
+            if "huggingface.co/" in url:
+                assert f"/resolve/{dataset.spec.revision}/" in url
+            else:
+                assert f"/{dataset.spec.revision}/" in url
+            assert "refs/heads/main" not in url
 
 
 def test_v2_controlled_datasets_have_vertical_family_coverage(tmp_path: Path) -> None:

From 08aedf760da4b3cdfcbea2bbb202ae81ad0ca491 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:56:09 +0200
Subject: [PATCH 32/33] test(mcb): account for JSON schema compliance task in
 registry contract

---
 .../model-capability-benchmark/tests/test_task_plugins.py     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/experiments/model-capability-benchmark/tests/test_task_plugins.py b/experiments/model-capability-benchmark/tests/test_task_plugins.py
index 150c3f32..3c902a34 100644
--- a/experiments/model-capability-benchmark/tests/test_task_plugins.py
+++ b/experiments/model-capability-benchmark/tests/test_task_plugins.py
@@ -29,10 +29,11 @@ def test_task_catalog_builds_without_runner_task_branches() -> None:
     registry = build_task_registry(TASKS)
 
     assert registry.summary() == {
-        "tasks": 5,
+        "tasks": 6,
         "task_ids": [
             "calibrated-intent-classification",
             "intent-classification",
+            "json-schema-compliance",
             "mathematical-reasoning",
             "qa-abstention",
             "structured-output",
@@ -40,6 +41,7 @@ def test_task_catalog_builds_without_runner_task_branches() -> None:
         "plugins": [
             "calibrated-intent-classification",
             "intent-classification",
+            "json-schema-compliance",
             "mathematical-reasoning",
             "qa-abstention",
             "structured-output",

From a8ed36ed65520f4c8df45dd97a735b81e2579107 Mon Sep 17 00:00:00 2001
From: Daniele21 <daniele21@users.noreply.github.com>
Date: Fri, 23 Oct 2026 12:58:51 +0200
Subject: [PATCH 33/33] ci(mcb): exercise public adapters, offline config, and
 live pinned dataset downloads

---
 .github/workflows/mcb-public-benchmarks.yml | 58 +++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 .github/workflows/mcb-public-benchmarks.yml

diff --git a/.github/workflows/mcb-public-benchmarks.yml b/.github/workflows/mcb-public-benchmarks.yml
new file mode 100644
index 00000000..c49ada2c
--- /dev/null
+++ b/.github/workflows/mcb-public-benchmarks.yml
@@ -0,0 +1,58 @@
+name: MCB public benchmark integration
+
+on:
+  pull_request:
+    paths:
+      - "experiments/model-capability-benchmark/**"
+      - "packages/benchmark-core/**"
+      - ".github/workflows/mcb-public-benchmarks.yml"
+  workflow_dispatch:
+
+jobs:
+  public-benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    defaults:
+      run:
+        working-directory: experiments/model-capability-benchmark
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        run: python -m pip install --disable-pip-version-check uv
+      - name: Sync locked dependencies
+        run: uv sync --frozen --extra dev
+      - name: Lint
+        run: uv run --frozen ruff check src tests/test_public_benchmarks.py
+      - name: Test public adapters and evaluators
+        run: uv run --frozen pytest -q tests/test_public_benchmarks.py
+      - name: Plan public Core without provider calls
+        run: uv run --frozen model-bench --suite suite_public.yaml plan --profile core --capabilities all
+      - name: Smoke-test real pinned upstream data without provider calls
+        run: |
+          uv run --frozen python - <<'PY'
+          from pathlib import Path
+          from benchmark_core import DatasetLoadContext
+          from model_capability_bench import load_capability_suite
+          suite = load_capability_suite(Path.cwd(), suite_filename="suite_public.yaml")
+          cache = Path.cwd() / ".cache/public-ci"
+          expected = {
+              "gsm8k-main-test": 12,
+              "squad-v2-validation": 12,
+              "jsonschemabench-github-easy-test": 12,
+          }
+          for dataset_id, count in expected.items():
+              result = suite.datasets.load(
+                  dataset_id,
+                  DatasetLoadContext(
+                      cache_dir=cache,
+                      profile=suite.profiles["smoke"],
+                      seed=42,
+                  ),
+              )
+              assert len(result.samples) == count, (dataset_id, len(result.samples))
+              assert result.source_checksums.get("source")
+              print(dataset_id, len(result.samples), result.available_count, result.selection_fingerprint)
+          PY