Add reproducible scientific and performance reports

.github/workflows/scientific-report.yml

@@ -0,0 +1,122 @@
+name: Scientific and performance report
+
+on:
+  workflow_dispatch:
+    inputs:
+      eri_filter:
+        description: Substring filter for ERI cases
+        required: true
+        default: h2-sto3g
+        type: string
+      mp2_sizes:
+        description: Comma-separated MP2 AO dimensions
+        required: true
+        default: '12,24'
+        type: string
+      mp2_memory:
+        description: Configured MP2 workspace budget
+        required: true
+        default: 32 MiB
+        type: string
+      include_heavy_eri:
+        description: Include the heavy ERI cases
+        required: true
+        default: false
+        type: boolean
+
+permissions:
+  contents: read
+
+jobs:
+  report:
+    name: Scientific validation and Criterion reports
+    runs-on: ubuntu-latest
+    env:
+      RUSTIQ_BENCH_FILTER: ${{ inputs.eri_filter }}
+      RUSTIQ_BENCH_HEAVY: ${{ inputs.include_heavy }}
+      RUSTIQ_MP2_SIZES: ${{ inputs.mp2_sizes }}
+      RUSTIQ_MP2_MEMORY: ${{ inputs.mp2_memory }}
+
+    steps:
+      - name: Check out selected commit
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+
+      - name: Install locked Python environment
+        run: uv sync --locked
+
+      - name: Remove stale Criterion measurements
+        run: rm -rf target/criterion
+
+      - name: Run canonical PySCF reference suite
+        id: reference
+        continue-on-error: true
+        run: uv run --locked pytest tools/reference
+
+      - name: Generate scientific report
+        id: scientific
+        if: always()
+        continue-on-error: true
+        run: uv run --locked python tools/reference/report.py --output-dir target/scientific-report
+
+      - name: Run selected ERI Criterion benchmarks
+        id: eri
+        continue-on-error: true
+        run: cargo bench --features bench-support --bench eri_timings
+
+      - name: Run selected MP2 Criterion benchmarks
+        id: mp2
+        continue-on-error: true
+        run: cargo bench --features bench-support --bench mp2_timings
+
+      - name: Generate performance report
+        id: performance
+        if: always()
+        continue-on-error: true
+        run: uv run --locked python tools/benchmarks/report.py --criterion-dir target/criterion --output-dir target/performance-report
+
+      - name: Upload generated reports
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: rustiq-scientific-performance-reports
+          if-no-files-found: warn
+          path: |
+            target/scientific-report/
+            target/performance-report/
+
+      - name: Write job summary
+        if: always()
+        env:
+          REFERENCE: ${{ steps.reference.outcome }}
+          SCIENTIFIC: ${{ steps.scientific.outcome }}
+          ERI: ${{ steps.eri.outcome }}
+          MP2: ${{ steps.mp2.outcome }}
+          PERFORMANCE: ${{ steps.performance.outcome }}
+        run: |
+          {
+            echo '# RustiQ scientific and performance reports'
+            echo
+            echo '| Stage | Result |'
+            echo '|---|---|'
+            echo "| Canonical PySCF suite | $REFERENCE |"
+            echo "| Scientific report | $SCIENTIFIC |"
+            echo "| ERI Criterion | $ERI |"
+            echo "| MP2 Criterion | $MP2 |"
+            echo "| Performance report | $PERFORMANCE |"
+            echo
+            echo 'Scientific validation and performance failures are reported independently. Download the artifact for detailed diagnostics.'
+          } >> "$GITHUB_STEP_SUMMARY"
+          if [[ "$REFERENCE" != success || "$SCIENTIFIC" != success ]]; then
+            echo 'Scientific validation failed; inspect the PySCF test and validation report steps.' >&2
+            exit 1
+          fi
+          if [[ "$ERI" != success || "$MP2" != success || "$PERFORMANCE" != success ]]; then
+            echo 'Performance reporting failed; inspect the Criterion and performance report steps.' >&2
+            exit 1
+          fi

benches/eri_timings.rs

@@ -59,6 +59,12 @@ fn eri_benchmarks(criterion: &mut Criterion) {
     let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     let basis_store = basis_store();
     let mut group = criterion.benchmark_group("eri");
+    let mut metadata = Vec::new();
+    let run_metadata = run_metadata();
+    let path = PathBuf::from("target/criterion/eri-metadata.json");
+    if path.exists() {
+        std::fs::remove_file(&path).expect("stale ERI metadata should be removable");
+    }
     for case in selected_cases() {
         // Ten is Criterion's minimum sample size; use it for heavy systems to
         // keep the requested measurement target from multiplying into hours.
@@ -82,6 +88,19 @@ fn eri_benchmarks(criterion: &mut Criterion) {
         let result = input
             .run_once()
             .expect("benchmark input should produce valid Coulomb self-integrals");
+        metadata.push(serde_json::json!({
+            "benchmark_id": format!("eri/{}/{}", case.name, case.basis),
+            "case": case.name,
+            "molecule": case.geometry,
+            "basis": case.basis,
+            "basis_functions": result.basis_functions,
+            "compact_integrals": result.compact_integrals,
+            "pair_count": result.pair_count,
+            "heavy": case.heavy,
+            "criterion_sample_size": if case.heavy { 10 } else { 30 },
+            "criterion_warm_up_seconds": if case.heavy { 1 } else { 2 },
+            "criterion_measurement_seconds": if case.heavy { 2 } else { 5 },
+        }));
         eprintln!(
             "ERI {}: {} basis functions, {} pairs, {} compact integrals, {:?} pair expansion, {:?} Schwarz bounds, {:?} fill, {} cached Coulomb values",
             case.name,
@@ -112,6 +131,57 @@ fn eri_benchmarks(criterion: &mut Criterion) {
     }
 
     group.finish();
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent).expect("Criterion metadata directory should be writable");
+    }
+    std::fs::write(
+        path,
+        serde_json::to_vec_pretty(&serde_json::json!({"run": run_metadata, "cases": metadata}))
+            .expect("metadata should serialize"),
+    )
+    .expect("Criterion metadata should be writable");
+}
+
+fn run_metadata() -> serde_json::Value {
+    let output = std::process::Command::new("git")
+        .args(["rev-parse", "HEAD"])
+        .output()
+        .ok();
+    let commit = output
+        .filter(|result| result.status.success())
+        .map(|result| String::from_utf8_lossy(&result.stdout).trim().to_owned());
+    let compiler = std::process::Command::new("rustc")
+        .arg("--version")
+        .output()
+        .ok();
+    let rustc = compiler
+        .filter(|result| result.status.success())
+        .map(|result| String::from_utf8_lossy(&result.stdout).trim().to_owned());
+    let clean = std::process::Command::new("git")
+        .args(["status", "--porcelain"])
+        .output()
+        .ok()
+        .filter(|result| result.status.success())
+        .map(|result| result.stdout.is_empty());
+    let cpu = std::fs::read_to_string("/proc/cpuinfo")
+        .ok()
+        .and_then(|info| {
+            info.lines()
+                .find(|line| line.starts_with("model name\t:"))
+                .map(str::to_owned)
+        })
+        .and_then(|line| {
+            line.split_once(':')
+                .map(|(_, value)| value.trim().to_owned())
+        });
+    let timestamp = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .ok()
+        .map(|duration| duration.as_secs_f64());
+    serde_json::json!({"git_commit": commit, "working_tree_clean": clean, "rustc_version": rustc,
+        "operating_system": std::env::consts::OS, "architecture": std::env::consts::ARCH,
+        "cpu_model": cpu, "benchmark_started_unix_seconds": timestamp,
+        "benchmark_profile": "cargo bench default bench profile"})
 }
 
 fn selected_cases() -> Vec<&'static BenchCase> {

benches/mp2_timings.rs

@@ -7,6 +7,10 @@ use rustiq_core::bench_support::Mp2BenchInput;
 use rustiq_core::config::MemoryLimit;
 
 fn mp2_benchmarks(criterion: &mut Criterion) {
+    let path = std::path::PathBuf::from("target/criterion/mp2-metadata.json");
+    if path.exists() {
+        std::fs::remove_file(&path).expect("stale MP2 metadata should be removable");
+    }
     let requested = std::env::var("RUSTIQ_MP2_MEMORY").unwrap_or_else(|_| "auto".into());
     let budget = if requested.eq_ignore_ascii_case("auto") {
         MemoryLimit::Auto.resolve()
@@ -18,6 +22,8 @@ fn mp2_benchmarks(criterion: &mut Criterion) {
         .split(',')
         .map(|size| size.parse().expect("integer AO dimension"))
         .collect();
+    let mut metadata = Vec::new();
+    let run_metadata = run_metadata();
 
     let mut group = criterion.benchmark_group("mp2");
     for n in sizes {
@@ -63,9 +69,73 @@ fn mp2_benchmarks(criterion: &mut Criterion) {
             budget.display().iec(),
             rayon::current_num_threads()
         );
+        metadata.push(serde_json::json!({
+            "ao_dimension": n,
+            "occupied_orbitals": occupied,
+            "workspace_budget_bytes": budget.as_u64(),
+            "block_size": memory.block_size,
+            "rayon_threads": rayon::current_num_threads(),
+            "dense_energy": dense_energy,
+            "blocked_energy": blocked_energy,
+            "energy_absolute_difference": (dense_energy - blocked_energy).abs(),
+            "criterion_sample_size": if n >= 48 { 10 } else { 30 },
+            "criterion_warm_up_seconds": if n >= 48 { 1 } else { 2 },
+            "criterion_measurement_seconds": if n >= 48 { 2 } else { 5 },
+        }));
     }
 
     group.finish();
+    if let Some(parent) = path.parent() {
+        std::fs::create_dir_all(parent).expect("Criterion metadata directory should be writable");
+    }
+    std::fs::write(
+        path,
+        serde_json::to_vec_pretty(&serde_json::json!({"run": run_metadata, "cases": metadata}))
+            .expect("metadata should serialize"),
+    )
+    .expect("Criterion metadata should be writable");
+}
+
+fn run_metadata() -> serde_json::Value {
+    let output = std::process::Command::new("git")
+        .args(["rev-parse", "HEAD"])
+        .output()
+        .ok();
+    let commit = output
+        .filter(|result| result.status.success())
+        .map(|result| String::from_utf8_lossy(&result.stdout).trim().to_owned());
+    let compiler = std::process::Command::new("rustc")
+        .arg("--version")
+        .output()
+        .ok();
+    let rustc = compiler
+        .filter(|result| result.status.success())
+        .map(|result| String::from_utf8_lossy(&result.stdout).trim().to_owned());
+    let clean = std::process::Command::new("git")
+        .args(["status", "--porcelain"])
+        .output()
+        .ok()
+        .filter(|result| result.status.success())
+        .map(|result| result.stdout.is_empty());
+    let cpu = std::fs::read_to_string("/proc/cpuinfo")
+        .ok()
+        .and_then(|info| {
+            info.lines()
+                .find(|line| line.starts_with("model name\t:"))
+                .map(str::to_owned)
+        })
+        .and_then(|line| {
+            line.split_once(':')
+                .map(|(_, value)| value.trim().to_owned())
+        });
+    let timestamp = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .ok()
+        .map(|duration| duration.as_secs_f64());
+    serde_json::json!({"git_commit": commit, "working_tree_clean": clean, "rustc_version": rustc,
+        "operating_system": std::env::consts::OS, "architecture": std::env::consts::ARCH,
+        "cpu_model": cpu, "benchmark_started_unix_seconds": timestamp,
+        "benchmark_profile": "cargo bench default bench profile"})
 }
 
 criterion_group!(benches, mp2_benchmarks);

docs/src/development/contributing.md

@@ -81,7 +81,7 @@ CLI TOML/Nickel adapter or another frontend
        -> iterative SCF and final canonicalization
        -> HfOutcome::Converged or HfOutcome::Unconverged
     -> converged HfSolution::mp2()
-       blocked AO-to-MO transformation and correlation energy
+       blocked AO-to-MO integral transformation and MP2 correlation-energy evaluation
 ```
 
 `prepare()` constructs reusable input data; it does not compute integrals or
@@ -143,9 +143,10 @@ cargo bench --bench mp2_timings --features bench-support
 cargo bench --bench eri_timings --features bench-support
 ```
 
-The MP2 benchmark compares dense and memory-blocked kernels on deterministic
-synthetic inputs. It validates that both paths produce matching energies before
-timing them; input and ERI construction remain outside the measured iterations.
+The MP2 benchmark compares dense and memory-blocked AO-to-MO transformations
+and their subsequent correlation-energy evaluation on deterministic synthetic
+inputs. It validates that both paths produce matching energies before timing
+them; input and ERI construction remain outside the measured iterations.
 `RUSTIQ_MP2_MEMORY`, `RUSTIQ_MP2_SIZES` (comma-separated AO dimensions), and
 `RAYON_NUM_THREADS` select the memory budget, AO dimensions, and thread count.
 The ERI benchmark times repeated integral construction for the checked-in
@@ -182,3 +183,53 @@ cargo bench --bench mp2_timings --features bench-support -- --baseline before
 See the [MP2 benchmark source](https://github.com/mveril/RustiQ/blob/main/benches/mp2_timings.rs)
 and [ERI benchmark source](https://github.com/mveril/RustiQ/blob/main/benches/eri_timings.rs).
 Benchmarks provide performance measurements, not scientific validation.
+
+### Generate reproducible reports
+
+From the repository root, the scientific report reuses the canonical PySCF
+reference cases and their absolute tolerances:
+
+```sh
+uv run --locked python tools/reference/report.py --output-dir target/scientific-report
+```
+
+It writes full precision results to `validation.json` and readable tables to
+`validation.md`. In addition to the cases run by `pytest tools/reference`, it
+includes the frozen-core MP2 variants exercised by those tests at 32 KiB and
+512 MiB workspace budgets. Total HF energies and MP2 correlation energies are
+reported separately. A case fails if either implementation does not converge,
+an energy comparison exceeds its existing tolerance, or the UHF spin expectation
+value misses its tolerance. Failed cases remain in the report, and the command
+returns a nonzero status. The report records software versions, commit and
+platform metadata so results can be compared across revisions.
+
+Generate Criterion measurements and their report with:
+
+```sh
+cargo bench --features bench-support --bench eri_timings
+cargo bench --features bench-support --bench mp2_timings
+uv run --locked python tools/benchmarks/report.py
+```
+
+The report is written to `target/performance-report/benchmarks.json` and
+`benchmarks.md`. To produce both reports on GitHub, open **Actions → Scientific
+and performance report → Run workflow** on the selected branch or commit. The
+workflow uploads `rustiq-scientific-performance-reports`; download that artifact
+from the completed workflow run to inspect the Markdown and JSON files.
+
+Criterion stores structured estimates in `target/criterion`; the benchmark
+executables also write the case dimensions and configuration used by the report
+generator there. The default runs include the small ERI cases and MP2 AO sizes
+12, 24, and 48. To select a smaller local run or opt into the expensive ERI
+cases, set `RUSTIQ_BENCH_FILTER`, `RUSTIQ_MP2_SIZES`, `RUSTIQ_MP2_MEMORY`, and
+`RUSTIQ_BENCH_HEAVY` as described above. The report includes Criterion means
+and confidence intervals. Its configured MP2 workspace is not measured peak
+RSS, and synthetic MP2 cases do not model complete molecular calculations.
+Results are machine-dependent; the on-demand GitHub Actions workflow starts
+with one small ERI case and AO sizes 12 and 24. Heavy ERI cases require explicit
+opt-in in the workflow.
+
+These reports provide numerical regression checks for selected cases. They do
+not establish general chemical accuracy; see the [scientific scope](../scientific-scope.md)
+for the limits of HF, MP2, and numerical agreement. Equal numeric convergence
+thresholds do not mean that RustiQ and PySCF use identical convergence tests.