Recover and consolidate benchmark runs with public dashboard exports

.github/workflows/mcb7-unified-runner-checks.yml

@@ -65,6 +65,7 @@ jobs:
           OPENAI_API_KEY: ci-placeholder
           MINICPM_API_KEY: ci-placeholder
           TYPESAFE_API_KEY: ci-placeholder
+          GEMINI_API_KEY: ci-placeholder
         run: |
           uv run model-bench models
           uv run model-bench tasks

experiments/model-capability-benchmark/HOW_TO_USE.md

@@ -501,6 +501,99 @@ benchmark semantics. Start a new campaign/run instead.
 
 ## 12. Run all five core verticals for one model
 
+### Consolidate runs by latest sample attempt
+
+`model-bench consolidate` writes a new immutable run from existing evidence,
+without inference. Repeat `--run-dir` for every base/recovery run to include:
+
+```bash
+uv run model-bench consolidate \
+  --run-dir results/runs/base-run \
+  --run-dir results/runs/recovery-run \
+  --run-id model-consolidated-v1 \
+  --run-group model-consolidated \
+  --plan-only
+```
+
+Remove `--plan-only` to materialize the snapshot and report. The output must not
+exist: use a new run ID for each new consolidation. Add all five base capability
+runs to retain a complete model result, then include recovery runs as additional
+sources. The source directory order does not affect selection.
+
+The logical key includes model, capability, dataset revision/split, sample ID,
+task version, prompt version and suite version. Selection uses the latest state
+timestamp; ties use source run ID, attempt and case ID in that order. A newer
+invalid result replaces an older valid result. Inference and evaluation are copied
+from the same attempt. No latest incomplete/failed attempt without paired evidence
+is silently dropped or replaced by an older success: consolidate rejects it and
+requires recovery first. Profiles, seeds, artifacts and dataset/task definitions
+must be compatible. Sources are checksumed before and after reading.
+
+Each selected case retains its original run/case/attempt, generation settings,
+timestamp and benchmark/runtime signatures. Aggregate metrics are recalculated on
+the deduplicated cases; original runs are untouched. Mixed generation budgets are
+explicit in the manifest and dashboard. Consolidated benchmark signatures are
+separate from uniform-budget comparisons. Hardware/resource evidence is not
+attributed to a single new execution of the consolidated cases.
+
+```bash
+uv run model-bench project --rebuild --export-dashboard
+uv run model-bench dashboard-build
+```
+
+Dashboard exports default to `--result-view latest`: the newest completed
+consolidated result replaces the base cells for that model/capability/profile;
+other models keep their ordinary CURRENT cells. Recovery subsets alone never
+replace complete results. Consolidated runs are labeled in the dashboard.
+To inspect only ordinary uniform-configuration results instead:
+
+```bash
+uv run model-bench dashboard-data --result-view standard
+uv run model-bench dashboard-build
+```
+
+The canonical SQL views `v_current_quality_results` and
+`v_current_performance_results` remain restricted to standard runs. The dashboard's
+latest selection is separately exposed as `v_latest_quality_results`.
+
+### Recover only invalid or failed cases with a different output budget
+
+Use `--rerun-errors-from` to select unsuccessful latest cases from a previous run.
+This includes `completed` cases whose evaluation has `valid=false`, as well as
+pipeline `failed` cases. Unlike `--retry-failures`, this creates a separate recovery
+experiment and can change the token budget. A newer valid result for the same
+model/capability/dataset/sample supersedes an earlier invalid result in the source
+evidence. Repeat the source option to combine multiple run directories.
+
+```bash
+export BENCHMARK_TIMEOUT_SECONDS=180
+
+uv run model-bench run \
+  --models "$MODEL" \
+  --capabilities structured-output \
+  --profile core \
+  --run-group "${CAMPAIGN}-recovery-4096" \
+  --run-id "${CAMPAIGN}-${MODEL}-structured-output-recovery-4096" \
+  --rerun-errors-from "results/runs/${CAMPAIGN}-${MODEL}-structured-output-core" \
+  --max-output-tokens 4096 \
+  --plan-only
+```
+
+Remove `--plan-only` to execute the selection. The preview does not contact a
+provider or create output files. Keep the source profile and seed; requested samples
+must still exist in the current dataset selection. Use a new run ID and output
+directory. Recovery manifests record source run IDs, state checksums, selected
+logical cases and the generation override; the original evidence is untouched.
+Repeating the same recovery command resumes it. If selection or token override
+changes, use another new run ID.
+
+Recovery runs remain available in history but are excluded from canonical CURRENT
+selection. Their scores measure recovery on an error-selected subset, not quality
+on the complete benchmark. The existing `BENCHMARK_TIMEOUT_SECONDS` environment
+variable controls provider request timeouts; more output tokens alone do not raise
+the timeout. `--max-output-tokens` also works for ordinary full runs without editing
+the shared suite configuration.
+
 After you are comfortable running the first vertical manually:
 
 ~~~bash
@@ -551,6 +644,55 @@ uv run model-bench dashboard-build
 
 ## 14. Move to the next model
 
+### One public JSON file for every dashboard view
+
+Every dashboard export automatically replaces
+`results/published/dashboard-data.json`. This is a single, versioned bundle with
+`overview`, `capabilities`, `models` and `runs` maps: all routes, including frontier
+and sensitivity (inside overview), can use the same snapshot. `generated_at_utc`
+identifies when it was exported. The source is the selected analytics view, so
+consolidated results and their mixed-budget labels are retained.
+
+Both commands refresh the public bundle automatically:
+
+```bash
+uv run model-bench project --export-dashboard
+uv run model-bench dashboard-data
+```
+
+The bundle is serialized completely and then atomically replaces the previous file.
+Failed exports preserve the previous public file. Only files referenced by the
+current export index are included; stale files are excluded. Known credential fields
+and local filesystem paths are redacted, including serialized event metadata.
+The original internal per-view exports remain available for local analysis.
+
+Host this file at a stable URL, for example `/data/dashboard-data.json`. To make the
+existing dashboard load all views dynamically, set this **before** its module script:
+
+```html
+<script>window.__MCB_DATA_URL__ = '/data/dashboard-data.json';</script>
+```
+
+The application fetches once on startup with `cache: 'no-store'`, validates the
+bundle envelope and initializes every route from it. A load error is displayed
+explicitly, without falling back to fixture data. Existing embedded/offline builds
+continue to work when no URL is configured. An external page can also consume it:
+
+```javascript
+const response = await fetch('/data/dashboard-data.json', { cache: 'no-store' });
+if (!response.ok) throw new Error(`Dashboard data: HTTP ${response.status}`);
+const snapshot = await response.json();
+const leaderboard = snapshot.overview.decision.model_summaries;
+const capabilities = snapshot.capabilities;
+const models = snapshot.models;
+const runs = snapshot.runs;
+```
+
+Updating the local bundle does not upload it: the hosting/deployment pipeline must
+sync `results/published/dashboard-data.json` to that URL. Serve JSON with compression
+and, if hosted on a different origin, appropriate CORS headers. Reloading the page
+loads the current snapshot; already-open pages do not poll automatically.
+
 Example:
 
 ~~~bash

experiments/model-capability-benchmark/README.md

@@ -165,9 +165,21 @@ New runs are stored by default under `results/runs/<run-id>`, while `run_group`
 Korgis model residency is managed through its admin API; the Korgis server process itself
 is intentionally external to this benchmark.
 
+`model-bench consolidate` creates an immutable latest-attempt result from base and
+recovery runs, deduplicating logical samples and recalculating metrics without
+inference. Dashboard exports prefer explicitly consolidated results by default and
+label mixed budgets; `--result-view standard` retains the uniform-configuration
+CURRENT view. Source evidence is preserved. See [the consolidation runbook](HOW_TO_USE.md#consolidate-runs-by-latest-sample-attempt).
+
 See `MCB_7_UNIFIED_RUNNER.md` for architecture, failure semantics and the controlled
 real-provider validation runbook.
 
+Dashboard exports also atomically update `results/published/dashboard-data.json`,
+a public bundle containing every view's data. Set `window.__MCB_DATA_URL__` before
+loading the dashboard to fetch that single file from a stable hosting URL.
+See [the public-data runbook](HOW_TO_USE.md#one-public-json-file-for-every-dashboard-view)
+for the schema, automatic refresh commands and hosting setup.
+
 ## Capability report
 
 Every run now produces a neutral capability × model report without an overall score or

experiments/model-capability-benchmark/dashboard/package.json

@@ -10,7 +10,8 @@
     "preview": "vite preview",
     "test:compare": "node --experimental-strip-types --test src/components/compareThreeLogic.test.ts src/components/hardwareEvidence.test.ts src/components/compareCockpitLogic.test.ts",
     "test:frontier": "node --experimental-strip-types --test src/components/frontierSelection.test.ts",
-    "test:guide": "node --experimental-strip-types --test src/datasetGuideData.test.ts"
+    "test:guide": "node --experimental-strip-types --test src/datasetGuideData.test.ts",
+    "test:bundle": "node --experimental-strip-types --test src/dashboardBundle.test.ts"
   },
   "dependencies": {
     "lucide-react": "^1.47.0",

experiments/model-capability-benchmark/dashboard/src/components/Shell.tsx

@@ -13,6 +13,7 @@ import {
   TrendingUp,
 } from 'lucide-react';
 import type { ReactNode } from 'react';
+import { overview } from '../data';
 import { navigate, usePathname } from '../router';
 
 function Link({
@@ -123,7 +124,20 @@ export function Shell({ children }: { children: ReactNode }) {
           </span>
         </div>
       </aside>
-      <main className="app-content">{children}</main>
+      <main className="app-content">
+        {overview.consolidations?.length ? (
+          <div className="fixture-banner" role="status">
+            Consolidated results · latest attempt per sample ·
+            {overview.consolidations.some((row) => row.generation_policy.mixed_generation)
+              ? ' mixed generation budgets; not a uniform-budget comparison. '
+              : ' combined source runs. '}
+            {[...new Set(overview.consolidations.map((row) => row.run_id))].map((runId) => (
+              <Link key={runId} href={`/runs/${encodeURIComponent(runId)}`}>{runId} </Link>
+            ))}
+          </div>
+        ) : null}
+        {children}
+      </main>
     </div>
   );
 }

experiments/model-capability-benchmark/dashboard/src/dashboardBundle.test.ts

@@ -0,0 +1,21 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { parseDashboardBundle } from './dashboardBundle.ts';
+
+const valid = {
+  schema_version: '1', generated_at_utc: '2026-10-22T00:00:00Z',
+  overview: { models: [], capabilities: [], cells: [], runs: [] },
+  capabilities: { cap: {} }, models: { model: {} }, runs: { run: {} },
+};
+
+test('accepts one snapshot containing all route maps', () => {
+  assert.equal(parseDashboardBundle(valid), valid);
+});
+
+test('rejects unsupported, missing or malformed route data', () => {
+  for (const invalid of [null, {}, { ...valid, schema_version: '2' },
+    { ...valid, models: [] }, { ...valid, runs: null }, { ...valid, capabilities: { cap: 'bad' } },
+    { ...valid, overview: { ...valid.overview, cells: 'bad' } }]) {
+    assert.throws(() => parseDashboardBundle(invalid), /Unsupported or incomplete/);
+  }
+});

experiments/model-capability-benchmark/dashboard/src/dashboardBundle.ts

@@ -0,0 +1,19 @@
+import type { DashboardBundle } from './types';
+
+function record(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+export function parseDashboardBundle(value: unknown): DashboardBundle {
+  if (!record(value) || value.schema_version !== '1' || typeof value.generated_at_utc !== 'string'
+    || !record(value.overview)) {
+    throw new Error('Unsupported or incomplete dashboard data file');
+  }
+  const overview = value.overview;
+  if (!['models', 'capabilities', 'cells', 'runs'].every((key) => Array.isArray(overview[key]))
+    || !['capabilities', 'models', 'runs'].every((key) => record(value[key])
+      && Object.values(value[key]).every(record))) {
+    throw new Error('Unsupported or incomplete dashboard data file');
+  }
+  return value as unknown as DashboardBundle;
+}

experiments/model-capability-benchmark/dashboard/src/data.ts

@@ -2,13 +2,16 @@ import overviewFixture from './data/overview.fixture.json';
 import capabilityFixture from './data/structured-output.fixture.json';
 import type {
   CapabilityPayload,
+  DashboardBundle,
   ModelPayload,
   OverviewPayload,
   RunPayload,
 } from './types';
 
 declare global {
   interface Window {
+    __MCB_DATA_URL__?: string;
+    __MCB_DASHBOARD_DATA__?: DashboardBundle;
     __MCB_OVERVIEW__?: OverviewPayload;
     __MCB_CAPABILITY__?: CapabilityPayload;
     __MCB_CAPABILITIES__?: Record<string, CapabilityPayload>;
@@ -18,15 +21,17 @@ declare global {
 }
 
 export const overview =
-  (window.__MCB_OVERVIEW__ ?? overviewFixture) as OverviewPayload;
+  (window.__MCB_DASHBOARD_DATA__?.overview ?? window.__MCB_OVERVIEW__ ?? overviewFixture) as OverviewPayload;
 
 export const capability =
-  (window.__MCB_CAPABILITY__ ?? capabilityFixture) as CapabilityPayload;
+  (window.__MCB_DASHBOARD_DATA__?.capabilities['structured-output']
+    ?? Object.values(window.__MCB_DASHBOARD_DATA__?.capabilities ?? {})[0]
+    ?? window.__MCB_CAPABILITY__ ?? capabilityFixture) as CapabilityPayload;
 
 export const capabilityPayloads: Record<string, CapabilityPayload> =
-  window.__MCB_CAPABILITIES__ ?? {
+  window.__MCB_DASHBOARD_DATA__?.capabilities ?? window.__MCB_CAPABILITIES__ ?? {
     [capability.capability_id]: capability,
   };
 
-export const modelPayloads = window.__MCB_MODELS__ ?? {};
-export const runPayloads = window.__MCB_RUNS__ ?? {};
+export const modelPayloads = window.__MCB_DASHBOARD_DATA__?.models ?? window.__MCB_MODELS__ ?? {};
+export const runPayloads = window.__MCB_DASHBOARD_DATA__?.runs ?? window.__MCB_RUNS__ ?? {};

experiments/model-capability-benchmark/dashboard/src/main.tsx

@@ -1,10 +1,23 @@
 import { StrictMode } from 'react';
 import { createRoot } from 'react-dom/client';
-import { App } from './App';
+import { parseDashboardBundle } from './dashboardBundle';
 import './styles.css';
 
-createRoot(document.getElementById('root')!).render(
-  <StrictMode>
-    <App />
-  </StrictMode>,
-);
+async function start() {
+  if (window.__MCB_DATA_URL__) {
+    const response = await fetch(window.__MCB_DATA_URL__, { cache: 'no-store' });
+    if (!response.ok) throw new Error(`Dashboard data: HTTP ${response.status}`);
+    window.__MCB_DASHBOARD_DATA__ = parseDashboardBundle(await response.json());
+  }
+  // Import after loading data: every route uses the same complete snapshot.
+  const { App } = await import('./App');
+  createRoot(document.getElementById('root')!).render(
+    <StrictMode><App /></StrictMode>,
+  );
+}
+
+start().catch((error: unknown) => {
+  const root = document.getElementById('root')!;
+  root.setAttribute('role', 'alert');
+  root.textContent = error instanceof Error ? error.message : 'Unable to load dashboard data';
+});

experiments/model-capability-benchmark/dashboard/src/pages/ExecutivePages.tsx

@@ -1221,7 +1221,7 @@ function ReliabilityBars({ models }: { models: DecisionModelSummary[] }) {
         return (
           <div className={tiedBest ? 'executive-reliability-row best' : 'executive-reliability-row'} key={model.model_signature}>
             <div>
-              <strong>{model.model_key}</strong>
+              <strong>{model.model_key}{model.result_policy === 'consolidated-latest-attempt' ? ' · consolidated' : ''}</strong>
               <span className="executive-model-meta">
                 <span>{model.quality_coverage_complete ? 'Complete quality coverage' : 'Partial quality coverage'}</span>
                 <RuntimeBadge model={model} compact />

experiments/model-capability-benchmark/dashboard/src/types.ts

@@ -90,6 +90,7 @@ export interface QualityPolicy {
 }
 
 export interface DecisionModelSummary extends DashboardModel {
+  result_policy?: 'standard' | 'consolidated-latest-attempt';
   quality_policy_id: string;
   overall_quality_score: number | null;
   quality_coverage_capabilities: number;
@@ -280,6 +281,13 @@ export interface SensitivityPayload {
 
 export interface OverviewPayload {
   schema_version: string;
+  result_view?: 'latest' | 'standard';
+  consolidations?: Array<{
+    run_id: string;
+    model_key: string;
+    capability_id: string;
+    generation_policy: { mixed_generation?: boolean; generation_configs?: unknown[] };
+  }>;
   fixture?: boolean;
   models: DashboardModel[];
   capabilities: string[];
@@ -290,6 +298,15 @@ export interface OverviewPayload {
   sensitivity?: SensitivityPayload;
 }
 
+export interface DashboardBundle {
+  schema_version: '1';
+  generated_at_utc: string;
+  overview: OverviewPayload;
+  capabilities: Record<string, CapabilityPayload>;
+  models: Record<string, ModelPayload>;
+  runs: Record<string, RunPayload>;
+}
+
 export interface FamilyBreakdown {
   model_key: string;
   family: string;