diff --git a/.github/workflows/regression-harness-unit.yaml b/.github/workflows/regression-harness-unit.yaml new file mode 100644 index 000000000..950ea51d5 --- /dev/null +++ b/.github/workflows/regression-harness-unit.yaml @@ -0,0 +1,46 @@ +name: Regression Harness Unit Tests + +# Unit tests for the regression harness's own logic (golden values, case loading), on +# synthetic data only. They load the harness project, not GPEC, so they run in minutes. +on: + pull_request: + branches: + - main + - develop + paths: + - 'regression-harness/**' + - '.github/workflows/regression-harness-unit.yaml' + push: + branches: + - main + - develop + paths: + - 'regression-harness/**' + - '.github/workflows/regression-harness-unit.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + unit: + name: regression-harness unit tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Julia + uses: julia-actions/setup-julia@v2 + with: + version: '1.11' + + - name: Instantiate the harness environment + run: julia --project=regression-harness -e 'using Pkg; Pkg.instantiate()' + + - name: Run the harness unit tests + run: julia --project=regression-harness regression-harness/test/runtests.jl diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md index e9246c962..60cb889ad 100644 --- a/docs/development/regression-harness.md +++ b/docs/development/regression-harness.md @@ -140,7 +140,23 @@ re-run once — those are exactly the entries whose provenance cannot be establi If the two refs in a comparison ran under different thread counts, the report flags it. +## Golden values + +`--refs` comparisons say whether a number changed; golden values say whether it still matches a committed, reviewed reference. Each case may have a golden file at `regression-harness/golden/.toml` holding every pinned quantity with its `rtol`, `atol`, tolerance class and the evidence behind the tolerance. + +```bash +regress --check --cases slayer_dels_fitzpatrick # working tree vs the committed goldens +regress --update-golden --cases slayer_dels_fitzpatrick --reason "..." # re-pin from a fresh run +``` + +- **Classes** come only from the `class` key that every `[quantities.*]` block in a case file must declare, never from the golden file or the quantity's name; a missing or unknown class makes the case fail to load. `topological` (exact), `equilibrium_scalar`, `physics_converged` gate; `diagnostic` and `unconverged` are reported only. +- **Tolerances** are either the provisional class default (`tolerance_basis = "class-default (provisional)"`) or `"measured"`, which requires a recorded `plateau_drift` or `platform_spread` no larger than `rtol`. No harness tool measures these yet, so every committed golden currently gates on provisional defaults, and `--check` labels those rows `provisional`. +- **`atol`** must be 0 on a gating entry unless its golden value contains an exact zero (a zero scalar, or an array element that is zero throughout); the relative test is the gate everywhere else. Loading and saving both enforce this. +- **Re-pinning** requires `--cases` and `--reason`, refuses a working tree with uncommitted tracked changes or with untracked files under `examples/`, `src/` or `regression-harness/cases/`, and prints an old → new line per moved quantity, flagging any move the old tolerance would have failed. +- **A move beyond the old tolerance** makes `--update-golden` refuse the whole re-pin unless `--accept-exceeding` is given. When it is, the entry loses its recorded evidence and becomes provisional, with a tolerance no looser than before, and the golden file's `[meta.exceeded.]` table records the previous version's value, the deviation and the old `rtol`/`atol`, so the reviewer sees each accepted move in the git diff. +- A failing `--check` is fixed by explaining the physics or fixing the regression, never by widening `rtol` or `atol`. The unit tests for this logic run with `julia --project=regression-harness regression-harness/test/runtests.jl` and need no GPEC build. + ## Exit status - `0` — every run completed (and, with `--fail-on-change`, nothing changed) -- `1` — a run failed, or a quantity changed under `--fail-on-change` +- `1` — a run failed, a quantity changed under `--fail-on-change`, or a `--check` golden comparison failed or gated nothing diff --git a/docs/development/toml-conventions.md b/docs/development/toml-conventions.md index 5ee2c3d5d..2949dbded 100644 --- a/docs/development/toml-conventions.md +++ b/docs/development/toml-conventions.md @@ -16,7 +16,8 @@ modify them. **Do not invent a new convention** — match what the existing exam files** — `examples/DIIID-like_ideal_example/gpec.toml` is the canonical reference for the common sections. Exception: in `regression-harness/cases/*.toml` the repeated `[quantities.*]` schema keys (`h5path`, `type`, `extract`, `label`, `noise_threshold`, - `order`) are developer metadata and do **not** need inline comments — a case file needs + `order`, and the required golden tolerance `class`) are developer metadata and do **not** + need inline comments (a `class` that is not the obvious one for its quantity should say why) — a case file needs the header plus an informative block comment per quantity group instead. 3. **Section comments only when informative.** No comment is required above a section. Keep a block comment only when it carries real information (e.g. the Solovev `[Wall]` note on why a diff --git a/regression-harness/cases/diiid_error_field.toml b/regression-harness/cases/diiid_error_field.toml index 2fd44b92a..4366f8483 100644 --- a/regression-harness/cases/diiid_error_field.toml +++ b/regression-harness/cases/diiid_error_field.toml @@ -16,6 +16,7 @@ extract = "all_real" label = "EFC overlap per kAt" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.efc_torque_full] h5path = "ErrorFields/NTV/torque_full_per_kat2" @@ -24,6 +25,7 @@ extract = "all_real" label = "EFC whole-field NTV torque per kAt² [N·m]" noise_threshold = 1e-8 order = 11 +class = "physics_converged" [quantities.efc_torque_residual] h5path = "ErrorFields/NTV/torque_residual_per_kat2" @@ -32,6 +34,7 @@ extract = "all_real" label = "EFC residual-field NTV torque per kAt² [N·m]" noise_threshold = 1e-8 order = 12 +class = "physics_converged" # The torque-versus-rotation table: reference rotation, the adaptive shift grid and the residual torque on it. [quantities.efc_omega_reference] @@ -41,6 +44,7 @@ extract = "all_real" label = "EFC reference rotation ω_ref [rad/s]" noise_threshold = 1e-6 order = 13 +class = "physics_converged" [quantities.efc_rotation_shift] h5path = "ErrorFields/NTV/RotationScan/rotation_shift" @@ -49,6 +53,7 @@ extract = "all_real" label = "EFC rotation-scan shifts [rad/s]" noise_threshold = 1e-6 order = 14 +class = "physics_converged" [quantities.efc_torque_residual_scan] h5path = "ErrorFields/NTV/RotationScan/torque_residual" @@ -57,6 +62,7 @@ extract = "all_real" label = "EFC residual torque against rotation [N·m/kAt²]" noise_threshold = 1e-8 order = 15 +class = "physics_converged" # Anchors of the stages feeding the couplings. [quantities.dominant_singular_values] @@ -66,6 +72,7 @@ extract = "all_real" label = "dominant-coupling singular values" noise_threshold = 1e-8 order = 20 +class = "physics_converged" [quantities.ef_delta_nominal] h5path = "ErrorFields/CoilSensitivities/DominantMode/delta_nominal" @@ -74,6 +81,7 @@ extract = "abs_first" label = "|delta_nominal| of coil set 1" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -81,3 +89,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" order = 99 +class = "diagnostic" diff --git a/regression-harness/cases/diiid_multi_n.toml b/regression-harness/cases/diiid_multi_n.toml index ca363248f..405234124 100644 --- a/regression-harness/cases/diiid_multi_n.toml +++ b/regression-harness/cases/diiid_multi_n.toml @@ -16,6 +16,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -24,6 +25,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" # Integration [quantities.nstep] @@ -33,6 +35,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" # Mode space [quantities.mpert] @@ -42,6 +45,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 40 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -50,6 +54,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 41 +class = "topological" [quantities.mode_n] h5path = "PerturbedEquilibrium/mode_n" @@ -58,6 +63,7 @@ extract = "checksum" label = "toroidal mode label of each mode index (checksum)" noise_threshold = 0 order = 42 +class = "physics_converged" # Control-surface response, one block per n (b~ space) [quantities.permeability_diag] @@ -67,6 +73,7 @@ extract = "diagonal_complex" label = "permeability diagonal P~_ii (all n blocks)" noise_threshold = 1e-8 order = 50 +class = "physics_converged" [quantities.forcing_b_rootarea] h5path = "PerturbedEquilibrium/forcing_b_root_area" @@ -75,6 +82,7 @@ extract = "all_complex" label = "forcing b~ (root-area-weighted), n = 1, 2" noise_threshold = 1e-10 order = 51 +class = "physics_converged" [quantities.response_b_rootarea] h5path = "PerturbedEquilibrium/response_b_root_area" @@ -83,6 +91,7 @@ extract = "all_complex" label = "response b~ (root-area-weighted), n = 1, 2" noise_threshold = 1e-8 order = 52 +class = "physics_converged" # Perturbed equilibrium energies and the Σ_n torque [quantities.pe_plasma_energy] @@ -92,6 +101,7 @@ extract = "value" label = "PE plasma energy (sum over n)" noise_threshold = 1e-10 order = 60 +class = "physics_converged" [quantities.pe_vacuum_energy] h5path = "PerturbedEquilibrium/Energies/vacuum_energy" @@ -100,6 +110,7 @@ extract = "value" label = "PE vacuum energy (sum over n)" noise_threshold = 1e-10 order = 61 +class = "physics_converged" [quantities.pe_surface_energy] h5path = "PerturbedEquilibrium/Energies/surface_energy" @@ -108,6 +119,7 @@ extract = "value" label = "PE surface energy (sum over n)" noise_threshold = 1e-10 order = 62 +class = "physics_converged" [quantities.pe_toroidal_torque] h5path = "PerturbedEquilibrium/Energies/toroidal_torque" @@ -116,6 +128,7 @@ extract = "value" label = "PE toroidal torque sum_n -2n Im(Phi_n' Lambda_nn^-1 Phi_n)/4" noise_threshold = 1e-12 order = 63 +class = "physics_converged" # Singular coupling: one row per resonant (surface, n); integer-q surfaces appear once per n [quantities.rational_n] @@ -125,6 +138,7 @@ extract = "all_real" label = "resonant n per row (rows sorted by psi across n)" noise_threshold = 0 order = 70 +class = "physics_converged" [quantities.rational_m] h5path = "PerturbedEquilibrium/SingularCoupling/rational_m" @@ -133,6 +147,7 @@ extract = "all_real" label = "resonant m per row" noise_threshold = 0 order = 71 +class = "physics_converged" [quantities.resonant_area_weighted_field_all] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -141,6 +156,7 @@ extract = "all_complex" label = "resonant area-weighted field b^r per row" noise_threshold = 1e-8 order = 72 +class = "physics_converged" [quantities.island_half_width] h5path = "PerturbedEquilibrium/SingularCoupling/island_half_width" @@ -149,6 +165,7 @@ extract = "all_real" label = "island half-widths per row" noise_threshold = 1e-8 order = 73 +class = "physics_converged" [quantities.chirikov_parameter] h5path = "PerturbedEquilibrium/SingularCoupling/chirikov_parameter" @@ -157,6 +174,7 @@ extract = "all_real" label = "Chirikov parameter per row" noise_threshold = 1e-8 order = 74 +class = "physics_converged" [quantities.delta_prime_all] h5path = "PerturbedEquilibrium/SingularCoupling/Delta_prime" @@ -165,6 +183,7 @@ extract = "all_complex" label = "forcing-driven Delta' per row" noise_threshold = 1e-8 order = 75 +class = "physics_converged" # Runtime [quantities.runtime] @@ -174,3 +193,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/diiid_n1.toml b/regression-harness/cases/diiid_n1.toml index 88ee87462..e115e8651 100644 --- a/regression-harness/cases/diiid_n1.toml +++ b/regression-harness/cases/diiid_n1.toml @@ -18,6 +18,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -26,6 +27,7 @@ extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -34,6 +36,7 @@ extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 order = 12 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -42,6 +45,7 @@ extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.vacuum_eigenvalue] h5path = "ForceFreeStates/FreeBoundaryStability/vacuum_eigenvalue" @@ -50,6 +54,7 @@ extract = "value" label = "vacuum matrix min eigenvalue" noise_threshold = 1e-10 order = 14 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.ep_all] @@ -59,6 +64,7 @@ extract = "all_complex" label = "plasma energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ev_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -67,6 +73,7 @@ extract = "all_complex" label = "vacuum energy (all)" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -75,6 +82,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 22 +class = "physics_converged" # Integration [quantities.nstep] @@ -84,6 +92,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -92,6 +101,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 31 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -101,6 +111,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 40 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -109,6 +120,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 41 +class = "equilibrium_scalar" [quantities.betat] h5path = "Equilibrium/beta_t" @@ -117,6 +129,7 @@ extract = "value" label = "beta_t" noise_threshold = 1e-12 order = 42 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -125,6 +138,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 43 +class = "equilibrium_scalar" [quantities.li1] h5path = "Equilibrium/l_i_1" @@ -133,6 +147,7 @@ extract = "value" label = "internal inductance li1" noise_threshold = 1e-12 order = 44 +class = "equilibrium_scalar" [quantities.li2] h5path = "Equilibrium/l_i_2" @@ -141,6 +156,7 @@ extract = "value" label = "internal inductance li2" noise_threshold = 1e-12 order = 45 +class = "equilibrium_scalar" [quantities.li3] h5path = "Equilibrium/l_i_3" @@ -149,6 +165,7 @@ extract = "value" label = "internal inductance li3" noise_threshold = 1e-12 order = 46 +class = "equilibrium_scalar" [quantities.betap1] h5path = "Equilibrium/beta_p_1" @@ -157,6 +174,7 @@ extract = "value" label = "poloidal beta betap1" noise_threshold = 1e-12 order = 47 +class = "equilibrium_scalar" [quantities.betap2] h5path = "Equilibrium/beta_p_2" @@ -165,6 +183,7 @@ extract = "value" label = "poloidal beta betap2" noise_threshold = 1e-12 order = 48 +class = "equilibrium_scalar" [quantities.betap3] h5path = "Equilibrium/beta_p_3" @@ -173,6 +192,7 @@ extract = "value" label = "poloidal beta betap3" noise_threshold = 1e-12 order = 49 +class = "equilibrium_scalar" [quantities.betaj] h5path = "Equilibrium/beta_j" @@ -181,6 +201,7 @@ extract = "value" label = "current beta betaj" noise_threshold = 1e-12 order = 57 +class = "equilibrium_scalar" [quantities.volume] h5path = "Equilibrium/volume" @@ -189,6 +210,7 @@ extract = "value" label = "plasma volume" noise_threshold = 1e-12 order = 58 +class = "equilibrium_scalar" [quantities.crnt] h5path = "Equilibrium/I_p" @@ -197,6 +219,7 @@ extract = "value" label = "plasma current" noise_threshold = 1e-12 order = 59 +class = "equilibrium_scalar" [quantities.bt0] h5path = "Equilibrium/B_T_axis" @@ -205,6 +228,7 @@ extract = "value" label = "toroidal field bt0" noise_threshold = 1e-12 order = 62 +class = "equilibrium_scalar" [quantities.bwall] h5path = "Equilibrium/B_T_wall" @@ -213,6 +237,7 @@ extract = "value" label = "wall field bwall" noise_threshold = 1e-12 order = 63 +class = "equilibrium_scalar" [quantities.aratio] h5path = "Equilibrium/aspect_ratio" @@ -221,6 +246,7 @@ extract = "value" label = "aspect ratio" noise_threshold = 1e-12 order = 64 +class = "equilibrium_scalar" [quantities.kappa] h5path = "Equilibrium/kappa" @@ -229,6 +255,7 @@ extract = "value" label = "elongation kappa" noise_threshold = 1e-12 order = 65 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -238,6 +265,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 50 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -246,6 +274,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 51 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -254,6 +283,7 @@ extract = "all_real" label = "singular q values" noise_threshold = 1e-8 order = 52 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -263,6 +293,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 60 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -271,6 +302,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 61 +class = "topological" [quantities.island_half_width] h5path = "PerturbedEquilibrium/SingularCoupling/island_half_width" @@ -279,6 +311,7 @@ extract = "all_real" label = "island half-widths" noise_threshold = 1e-8 order = 81 +class = "physics_converged" [quantities.chirikov_parameter] h5path = "PerturbedEquilibrium/SingularCoupling/chirikov_parameter" @@ -287,6 +320,7 @@ extract = "all_real" label = "Chirikov parameter" noise_threshold = 1e-8 order = 82 +class = "physics_converged" [quantities.resonant_area_weighted_field] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -295,6 +329,7 @@ extract = "norm" label = "||resonant area-weighted field||" noise_threshold = 1e-8 order = 83 +class = "physics_converged" [quantities.resonant_area_weighted_field_all] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -302,6 +337,7 @@ type = "complex_vector" extract = "all_complex" label = "resonant area-weighted field b^r" noise_threshold = 1e-8 +class = "physics_converged" # Dominant resonant-coupling mode: singular values of the b̃-space coupling matrix over every # rational surface (coordinate-invariant), and the applied forcing's magnitude on the dominant @@ -313,6 +349,7 @@ extract = "all_real" label = "dominant-coupling singular values" noise_threshold = 1e-8 order = 84 +class = "physics_converged" [quantities.dominant_forcing_overlap] h5path = "PerturbedEquilibrium/SingularCoupling/DominantMode/forcing_overlap" @@ -321,6 +358,7 @@ extract = "abs_first" label = "|forcing overlap with dominant mode|" noise_threshold = 1e-8 order = 85 +class = "physics_converged" # Perturbed equilibrium: energies [quantities.pe_plasma_energy] @@ -330,6 +368,7 @@ extract = "value" label = "PE plasma energy" noise_threshold = 1e-10 order = 91 +class = "physics_converged" [quantities.pe_vacuum_energy] h5path = "PerturbedEquilibrium/Energies/vacuum_energy" @@ -338,6 +377,7 @@ extract = "value" label = "PE vacuum energy" noise_threshold = 1e-10 order = 92 +class = "physics_converged" [quantities.pe_surface_energy] h5path = "PerturbedEquilibrium/Energies/surface_energy" @@ -346,6 +386,7 @@ extract = "value" label = "PE surface energy" noise_threshold = 1e-10 order = 93 +class = "physics_converged" [quantities.pe_toroidal_torque] h5path = "PerturbedEquilibrium/Energies/toroidal_torque" @@ -354,6 +395,7 @@ extract = "value" label = "PE toroidal torque" noise_threshold = 1e-12 order = 94 +class = "physics_converged" # NTV (neoclassical toroidal viscosity) toroidal torque T_φ [N·m] from the # [KineticForces] FGAR method (pressure moment) — a diagnostic computed on top of @@ -367,6 +409,7 @@ extract = "value" label = "NTV torque FGAR [N·m]" noise_threshold = 1e-2 order = 95 +class = "physics_converged" # NTV kinetic energy δW_k = Im(T)/(2n) [J] through the same post-PE FGAR path. # Tracked at a tighter threshold — it isolates the energy component the @@ -378,6 +421,7 @@ extract = "value" label = "NTV kinetic energy dW FGAR [J]" noise_threshold = 1e-3 order = 96 +class = "physics_converged" # Profiles (checksums) [quantities.q_profile] @@ -387,6 +431,7 @@ extract = "checksum" label = "q profile (checksum)" noise_threshold = 0 order = 70 +class = "physics_converged" [quantities.pressure_profile] h5path = "Equilibrium/Profiles/mu0p" @@ -395,6 +440,7 @@ extract = "checksum" label = "pressure profile (checksum)" noise_threshold = 0 order = 71 +class = "physics_converged" # Local stability profiles (local_stability_flag = true). Checksums hash the full # profile so they are robust to NaN edge-surface entries and catch any change. @@ -405,6 +451,7 @@ extract = "checksum" label = "Mercier D_I profile (checksum)" noise_threshold = 0 order = 72 +class = "physics_converged" [quantities.locstab_dr] h5path = "LocalStability/D_R" @@ -413,6 +460,7 @@ extract = "checksum" label = "resistive interchange D_R profile (checksum)" noise_threshold = 0 order = 73 +class = "physics_converged" [quantities.locstab_ballooning_dp] h5path = "LocalStability/ballooning_Delta_prime" @@ -421,6 +469,7 @@ extract = "checksum" label = "ballooning Delta' profile (checksum)" noise_threshold = 0 order = 74 +class = "physics_converged" # Error fields — per-coil-set overlap with the full-window dominant mode and its rigid shift/tilt # sensitivities. Singular vectors carry an arbitrary phase, so only magnitudes are tracked; the @@ -433,6 +482,7 @@ extract = "abs_first" label = "|delta_nominal| of coil set 1" noise_threshold = 1e-10 order = 86 +class = "physics_converged" [quantities.ef_shift_sensitivity] h5path = "ErrorFields/CoilSensitivities/DominantMode/shift_sensitivity" @@ -441,6 +491,7 @@ extract = "norm" label = "||ddelta/d(shift)|| over coil sets" noise_threshold = 1e-10 order = 87 +class = "physics_converged" [quantities.ef_tilt_sensitivity] h5path = "ErrorFields/CoilSensitivities/DominantMode/tilt_sensitivity" @@ -449,6 +500,7 @@ extract = "norm" label = "||ddelta/d(tilt)|| over coil sets" noise_threshold = 1e-10 order = 88 +class = "physics_converged" # Runtime [quantities.runtime] @@ -458,6 +510,7 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" # Root-area-weighted (coordinate-invariant) forcing field b̃ on the control surface: the # applied coil spectrum pushed through the √(J|∇ψ|) weight operator. Tracks the field-space @@ -469,3 +522,4 @@ extract = "norm" label = "||forcing b~|| (root-area-weighted)" noise_threshold = 1e-10 order = 1000 +class = "physics_converged" diff --git a/regression-harness/cases/diiid_n1_riccati.toml b/regression-harness/cases/diiid_n1_riccati.toml index f41d9ed0a..9b8442317 100644 --- a/regression-harness/cases/diiid_n1_riccati.toml +++ b/regression-harness/cases/diiid_n1_riccati.toml @@ -10,23 +10,38 @@ name = "diiid_n1_riccati" description = "DIII-D-like equilibrium, n=1, Riccati integrator Δ' matrix" example_dir = "examples/DIIID-like_riccati_deltaprime_example" +# Which formalism produced the Δ′ below. The Δ′ matrix is written to one canonical +# SingularSurfaces/ path whichever integrator computed it, so the numbers alone cannot say +# whether a gold pinned from the BVP is being compared against a Galerkin run. Pinning the +# deck's declared choice makes a formalism swap a loud failure instead of a silent re-gold. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 +class = "topological" + # Tearing stability Δ' — the msing×msing BVP matrix diagonal, and the raw side-major # outer-region matrix it is projected from. [quantities.delta_prime] h5path = "SingularSurfaces/Delta_prime_matrix" type = "complex_matrix" extract = "diagonal_complex" -label = "delta prime (BVP diagonal)" +label = "Δ′ STRIDE-like BVP (PEST3) diagonal" noise_threshold = 1e-8 order = 10 +class = "physics_converged" [quantities.delta_prime_raw] h5path = "SingularSurfaces/Delta_prime_raw" type = "complex_matrix" extract = "all_complex" -label = "delta prime (raw side-major)" +label = "Δ′ STRIDE-like BVP (PEST3) raw side-major" noise_threshold = 1e-8 order = 11 +class = "physics_converged" [quantities.delta_coil] h5path = "SingularSurfaces/Delta_coil" @@ -35,6 +50,7 @@ extract = "all_complex" label = "edge coil response delta_coil" noise_threshold = 1e-8 order = 12 +class = "physics_converged" # Energies — leading generalized (W,N) pencil eigenvalues at the final truncation (psilim): power-normalized (⟨|ξ|²⟩ = 1 metric) and invariant to the working-coordinate Jacobian. [quantities.et_real] @@ -44,6 +60,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -52,6 +69,7 @@ extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -60,6 +78,7 @@ extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 order = 22 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -68,6 +87,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 23 +class = "physics_converged" # Singular surfaces — the Δ' matrix is indexed by these, so a shift here explains a Δ' shift. [quantities.msing] @@ -77,6 +97,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 30 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -85,6 +106,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 31 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -93,6 +115,7 @@ extract = "all_real" label = "singular q values" noise_threshold = 1e-8 order = 32 +class = "physics_converged" # Integration — chunk-endpoint step counts move when the chunk sizing changes. [quantities.nstep] @@ -102,6 +125,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 40 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -110,6 +134,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 41 +class = "diagnostic" # Mode numbers [quantities.mpert] @@ -119,6 +144,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 50 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -127,6 +153,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 51 +class = "topological" # Equilibrium — shared with diiid_n1; confirms the two cases sit on the same equilibrium. [quantities.q0] @@ -136,6 +163,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 60 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -144,6 +172,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 61 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -152,6 +181,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 62 +class = "equilibrium_scalar" # Runtime [quantities.runtime] @@ -161,3 +191,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/diiid_slayer_n1.toml b/regression-harness/cases/diiid_slayer_n1.toml index 90edf2e09..9824a728c 100644 --- a/regression-harness/cases/diiid_slayer_n1.toml +++ b/regression-harness/cases/diiid_slayer_n1.toml @@ -10,6 +10,18 @@ name = "diiid_slayer_n1" description = "DIII-D-like H-mode equilibrium, n=1, SLAYER tearing-mode analysis (uncoupled per-surface, AMR, validity-gated)" example_dir = "examples/DIIID-like_SLAYER_example" +# Which formalism produced the Δ′ this case's layer physics consumes. The Δ′ matrix lands on +# one canonical SingularSurfaces/ path whichever integrator computed it, so pinning the deck's +# declared choice is what keeps a BVP-derived gold from being compared against a Galerkin run. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 +class = "topological" + # Per-surface SLAYER layer parameters (geometry + dimensionless) [quantities.slayer_ising] h5path = "Tearing/PerSurface/rational_index" @@ -18,6 +30,7 @@ extract = "all_real" label = "SLAYER surface indices" noise_threshold = 0 order = 10 +class = "physics_converged" [quantities.slayer_m] h5path = "Tearing/PerSurface/m" @@ -26,6 +39,7 @@ extract = "all_real" label = "SLAYER poloidal m" noise_threshold = 0 order = 11 +class = "physics_converged" [quantities.slayer_n] h5path = "Tearing/PerSurface/n" @@ -34,6 +48,7 @@ extract = "all_real" label = "SLAYER toroidal n" noise_threshold = 0 order = 12 +class = "physics_converged" [quantities.slayer_rs] h5path = "Tearing/PerSurface/rs" @@ -42,6 +57,7 @@ extract = "all_real" label = "SLAYER minor radius rs" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.slayer_sval_r] h5path = "Tearing/PerSurface/sval_r" @@ -50,6 +66,7 @@ extract = "all_real" label = "SLAYER r-based shear" noise_threshold = 1e-10 order = 14 +class = "physics_converged" [quantities.slayer_lu] h5path = "Tearing/PerSurface/lu" @@ -58,6 +75,7 @@ extract = "all_real" label = "SLAYER Lundquist S" noise_threshold = 1e-8 order = 15 +class = "physics_converged" [quantities.slayer_D_norm] h5path = "Tearing/PerSurface/D_norm" @@ -66,6 +84,7 @@ extract = "all_real" label = "SLAYER D_norm" noise_threshold = 1e-10 order = 16 +class = "physics_converged" [quantities.slayer_P_perp] h5path = "Tearing/PerSurface/P_perp" @@ -74,6 +93,7 @@ extract = "all_real" label = "SLAYER P_perp" noise_threshold = 1e-8 order = 17 +class = "physics_converged" [quantities.slayer_tauk] h5path = "Tearing/PerSurface/tau_k" @@ -82,6 +102,7 @@ extract = "all_real" label = "SLAYER tauk" noise_threshold = 1e-12 order = 18 +class = "physics_converged" [quantities.slayer_iota_e] h5path = "Tearing/PerSurface/iota_e" @@ -90,6 +111,7 @@ extract = "all_real" label = "SLAYER iota_e" noise_threshold = 1e-12 order = 19 +class = "physics_converged" # Normalized diamagnetic frequencies. iota_e above is their ratio, so it is blind # to a common factor on both; these pin the drive itself. @@ -100,6 +122,7 @@ extract = "all_real" label = "SLAYER Q_e" noise_threshold = 1e-12 order = 20 +class = "physics_converged" [quantities.slayer_Q_i] h5path = "Tearing/PerSurface/Q_i" @@ -108,6 +131,7 @@ extract = "all_real" label = "SLAYER Q_i" noise_threshold = 1e-12 order = 21 +class = "physics_converged" # Tearing eigenvalue (coupled mode → length 1). The headline deliverable: # a real, nonzero growth rate on a realistic equilibrium. Root-extraction @@ -125,6 +149,7 @@ extract = "first_3_complex" label = "SLAYER Q_root [2/1,3/1,4/1]" noise_threshold = 1e-4 order = 30 +class = "physics_converged" [quantities.slayer_omega_Hz] h5path = "Tearing/Roots/omega" @@ -133,6 +158,7 @@ extract = "first_3" label = "SLAYER ω_Hz [2/1,3/1,4/1]" noise_threshold = 1.0 order = 32 +class = "physics_converged" [quantities.slayer_gamma_Hz] h5path = "Tearing/Roots/gamma" @@ -141,6 +167,7 @@ extract = "first_3" label = "SLAYER γ_Hz [2/1,3/1,4/1]" noise_threshold = 1e-1 order = 33 +class = "physics_converged" # no_root flag (1 = extraction failed). Pinned for the inner three surfaces; # this case guards that the 2/1, 3/1, 4/1 keep finding a real root. @@ -151,6 +178,58 @@ extract = "first_3" label = "SLAYER no_root flags [2/1,3/1,4/1]" noise_threshold = 0 order = 34 +class = "physics_converged" + +# Outer-region Δ' consumed by the SLAYER matching — the same STRIDE-like BVP (PEST3) matrix +# the riccati case pins at source; tracked here too so a change in what SLAYER actually +# ingests is caught even if the source case is not run. +[quantities.slayer_delta_prime_diag] +h5path = "Tearing/PerSurface/Delta_prime_matrix" +type = "complex_matrix" +extract = "diagonal_complex" +label = "Δ′ STRIDE-like BVP (PEST3) diagonal, as ingested by SLAYER" +noise_threshold = 1e-8 +order = 40 +class = "physics_converged" + +# Inner-layer solve outputs per surface (the resistive-layer Riccati solution itself, not the +# outer matching). The adaptive-scan Δ(Q) sample arrays are deliberately NOT pinned: AMR sample +# locations move under any refinement change, so they have no stable identity to pin. +[quantities.slayer_delta_s] +h5path = "Tearing/LayerWidths/delta_s" +type = "complex_vector" +extract = "all_complex" +label = "SLAYER inner-layer thickness δ_s (complex, Riccati)" +noise_threshold = 1e-10 +order = 41 +class = "physics_converged" + +[quantities.slayer_delta_s_over_d_beta] +h5path = "Tearing/LayerWidths/delta_s_over_d_beta" +type = "complex_vector" +extract = "all_complex" +label = "SLAYER inner-layer δ_s/d_β (complex, dimensionless)" +noise_threshold = 1e-10 +order = 42 +class = "physics_converged" + +[quantities.slayer_delta_s_abs] +h5path = "Tearing/LayerWidths/delta_s_abs" +type = "real_vector" +extract = "all_real" +label = "SLAYER inner-layer physical thickness |δ_s| [m]" +noise_threshold = 1e-12 +order = 43 +class = "physics_converged" + +[quantities.slayer_d_beta] +h5path = "Tearing/LayerWidths/d_beta" +type = "real_vector" +extract = "all_real" +label = "SLAYER β-weighted ion drift scale d_β [m]" +noise_threshold = 1e-12 +order = 44 +class = "physics_converged" # Settings (catches accidental config drift) [quantities.slayer_enabled] @@ -160,6 +239,7 @@ extract = "value" label = "SLAYER enabled flag" noise_threshold = 0 order = 90 +class = "topological" [quantities.runtime] h5path = "" @@ -168,3 +248,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/efit_fixedbdy_separatrix.toml b/regression-harness/cases/efit_fixedbdy_separatrix.toml index d64f09ba7..323150732 100644 --- a/regression-harness/cases/efit_fixedbdy_separatrix.toml +++ b/regression-harness/cases/efit_fixedbdy_separatrix.toml @@ -19,6 +19,7 @@ extract = "value" label = "psio (total poloidal flux)" noise_threshold = 1e-12 order = 10 +class = "equilibrium_scalar" [quantities.q0] h5path = "Equilibrium/q_axis" @@ -27,6 +28,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 11 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -35,6 +37,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 12 +class = "equilibrium_scalar" [quantities.betat] h5path = "Equilibrium/beta_t" @@ -43,6 +46,7 @@ extract = "value" label = "beta_t" noise_threshold = 1e-12 order = 13 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -51,6 +55,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 14 +class = "equilibrium_scalar" [quantities.runtime] h5path = "" @@ -59,3 +64,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/gal_resistive_diiid.toml b/regression-harness/cases/gal_resistive_diiid.toml index b13388d8a..6af81496a 100644 --- a/regression-harness/cases/gal_resistive_diiid.toml +++ b/regression-harness/cases/gal_resistive_diiid.toml @@ -9,6 +9,19 @@ name = "gal_resistive_diiid" description = "DIII-D-like, n=1, RDCON outer-region Galerkin Δ′ with rpec coil columns (delta_coil)" example_dir = "examples/DIIID-like_gal_resistive_example" +# Which formalism produced the Galerkin Δ′ below. The Δ′ matrix is written to one canonical +# SingularSurfaces/ path whichever integrator computed it, so the numbers alone cannot say whether +# a gold pinned from the Galerkin solve is being compared against a BVP run. Pinning the deck's +# declared choice makes a formalism swap a loud failure instead of a silent re-gold. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 +class = "topological" + # Number of resonant surfaces in the gal domain [quantities.gal_msing] h5path = "ForceFreeStates/Solutions/GalerkinIntegration/rational_count" @@ -17,6 +30,7 @@ extract = "value" label = "gal # singular surfaces" noise_threshold = 0 order = 10 +class = "topological" [quantities.gal_sing_q] h5path = "ForceFreeStates/Solutions/GalerkinIntegration/rational_q" @@ -25,24 +39,27 @@ extract = "all_real" label = "gal singular q values" noise_threshold = 1e-8 order = 11 +class = "physics_converged" # PEST-3 Δ matching matrix — per-surface diagonal (the physics-meaningful tearing Δ′) [quantities.gal_pest3_delta_diag] h5path = "SingularSurfaces/Delta_prime_matrix" type = "complex_matrix" extract = "diagonal_complex" -label = "gal PEST3 Δ diagonal" +label = "Δ′ Galerkin RDCON (PEST3) diagonal" noise_threshold = 1e-6 order = 20 +class = "physics_converged" # Raw outer Δ′ matrix (2·msing × 2·msing) — Frobenius norm catches any element drift [quantities.gal_delta_norm] h5path = "SingularSurfaces/Delta_prime_raw" type = "complex_matrix" extract = "norm" -label = "||gal Δ′ matrix||" +label = "||Δ′ Galerkin RDCON (PEST3) matrix||" noise_threshold = 1e-6 order = 21 +class = "physics_converged" # rpec coil-response block (edge mode × surface-side) — Frobenius norm [quantities.gal_delta_coil_norm] @@ -52,6 +69,7 @@ extract = "norm" label = "||gal Δ_coil block||" noise_threshold = 1e-6 order = 22 +class = "physics_converged" # Mercier index per surface [quantities.gal_di] @@ -61,6 +79,7 @@ extract = "all_real" label = "gal D_I per surface" noise_threshold = 1e-8 order = 30 +class = "physics_converged" # Resonant exponents α per surface [quantities.gal_alpha] @@ -70,6 +89,7 @@ extract = "all_complex" label = "gal α per surface" noise_threshold = 1e-8 order = 31 +class = "physics_converged" # --- DRIVEN (RPEC) inner-layer matching (gal_match_flag) --- # Only finite, well-defined matching quantities are tracked. The reconstructed ξ/ξ′ are deliberately @@ -85,6 +105,7 @@ extract = "norm" label = "||gal inner-layer Δ||" noise_threshold = 1e-6 order = 41 +class = "physics_converged" # Outer-region matched coefficients cout (2·msing × mcoil) — the 4·msing matching assembly + solve. [quantities.gal_match_cout_norm] @@ -94,6 +115,7 @@ extract = "norm" label = "||gal match cout||" noise_threshold = 1e-6 order = 42 +class = "physics_converged" # Matching linear-solve residual ‖mat·cof − rmat‖/‖rmat‖ — health check (should stay ~machine eps). [quantities.gal_match_residual] @@ -103,6 +125,7 @@ extract = "value" label = "gal match residual" noise_threshold = 1e-10 order = 44 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -111,3 +134,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/gal_resistive_pe.toml b/regression-harness/cases/gal_resistive_pe.toml index 9a04b30ac..27d4b24d4 100644 --- a/regression-harness/cases/gal_resistive_pe.toml +++ b/regression-harness/cases/gal_resistive_pe.toml @@ -13,6 +13,19 @@ example_dir = "examples/DIIID-like_gal_resistive_pe_example" # FINITE, per-rational-surface singular-coupling scalars (the physics-meaningful driven quantities) — not # any whole-profile ξ/ξ′ norm (those diverge at the rationals and are ill-posed as benchmarks). +# Which formalism produced the Galerkin Δ′ below. The Δ′ matrix is written to one canonical +# SingularSurfaces/ path whichever integrator computed it, so the numbers alone cannot say whether +# a gold pinned from the Galerkin solve is being compared against a BVP run. Pinning the deck's +# declared choice makes a formalism swap a loud failure instead of a silent re-gold. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 +class = "topological" + # Surface locations / count (sanity) [quantities.pe_rational_q] h5path = "PerturbedEquilibrium/SingularCoupling/rational_q" @@ -21,6 +34,7 @@ extract = "all_real" label = "rational q values" noise_threshold = 1e-8 order = 10 +class = "physics_converged" # Island half-widths per rational surface — the headline driven quantity [quantities.pe_island_half_width] @@ -30,6 +44,7 @@ extract = "all_real" label = "island half-widths" noise_threshold = 1e-8 order = 20 +class = "physics_converged" # Chirikov overlap parameter per surface [quantities.pe_chirikov] @@ -39,6 +54,7 @@ extract = "all_real" label = "Chirikov parameter" noise_threshold = 1e-8 order = 21 +class = "physics_converged" # Resonant area-weighted field b̄^r = Φ^r/A^r per surface (tesla) — norm + per-surface complex values [quantities.pe_resonant_flux_norm] @@ -48,6 +64,7 @@ extract = "norm" label = "||resonant b field||" noise_threshold = 1e-6 order = 30 +class = "physics_converged" [quantities.pe_resonant_flux_all] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -56,6 +73,7 @@ extract = "all_complex" label = "resonant b field (all)" noise_threshold = 1e-8 order = 31 +class = "physics_converged" # Penetrated (reconnected) resonant field per surface — the resistive shielding signature [quantities.pe_penetrated_field] @@ -65,15 +83,19 @@ extract = "all_complex" label = "penetrated b field" noise_threshold = 1e-8 order = 32 +class = "physics_converged" -# Per-surface Δ' from the PE singular-coupling analysis +# Per-surface driven-response Δ' from the PE singular-coupling jump (∂b^ψ/∂ψ across the +# rational surface over 2π·χ₁, forced by the applied flux). This is a driven-response +# diagnostic, NOT the tearing-stability Δ' of the STRIDE-like BVP or Galerkin RDCON solvers. [quantities.pe_delta_prime] h5path = "PerturbedEquilibrium/SingularCoupling/Delta_prime" type = "complex_vector" extract = "all_complex" -label = "PE Δ' per surface" +label = "PE driven-response Δ′ (SingularCoupling jump)" noise_threshold = 1e-8 order = 33 +class = "physics_converged" # Resonant-field coupling matrix (n_rational × numpert_total) — Frobenius norm, catches broader drift [quantities.pe_C_resonant_flux_norm] @@ -83,6 +105,7 @@ extract = "norm" label = "||C resonant b field||" noise_threshold = 1e-6 order = 40 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -91,3 +114,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/ggj_ray_q500i.toml b/regression-harness/cases/ggj_ray_q500i.toml index f5722ee4d..3b33b87f4 100644 --- a/regression-harness/cases/ggj_ray_q500i.toml +++ b/regression-harness/cases/ggj_ray_q500i.toml @@ -22,6 +22,7 @@ extract = "value" label = "Re(Δ_odd)" noise_threshold = 1e-4 order = 10 +class = "physics_converged" [quantities.delta_odd_imag] h5path = "ggj/delta_odd_imag" @@ -30,6 +31,7 @@ extract = "value" label = "Im(Δ_odd)" noise_threshold = 1e-4 order = 11 +class = "physics_converged" [quantities.delta_even_real] h5path = "ggj/delta_even_real" @@ -38,6 +40,7 @@ extract = "value" label = "Re(Δ_even)" noise_threshold = 1e-4 order = 12 +class = "physics_converged" [quantities.delta_even_imag] h5path = "ggj/delta_even_imag" @@ -46,6 +49,7 @@ extract = "value" label = "Im(Δ_even)" noise_threshold = 1e-4 order = 13 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -54,3 +58,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/ggj_reference.toml b/regression-harness/cases/ggj_reference.toml index b35ca20a7..a8df484d9 100644 --- a/regression-harness/cases/ggj_reference.toml +++ b/regression-harness/cases/ggj_reference.toml @@ -15,6 +15,7 @@ extract = "value" label = "Re(Δ_odd)" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.delta_odd_imag] h5path = "ggj/delta_odd_imag" @@ -23,6 +24,7 @@ extract = "value" label = "Im(Δ_odd)" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.delta_even_real] h5path = "ggj/delta_even_real" @@ -31,6 +33,7 @@ extract = "value" label = "Re(Δ_even)" noise_threshold = 1e-10 order = 12 +class = "physics_converged" [quantities.delta_even_imag] h5path = "ggj/delta_even_imag" @@ -39,6 +42,7 @@ extract = "value" label = "Im(Δ_even)" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -47,3 +51,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/slayer_dels_fitzpatrick.toml b/regression-harness/cases/slayer_dels_fitzpatrick.toml new file mode 100644 index 000000000..c44dcb135 --- /dev/null +++ b/regression-harness/cases/slayer_dels_fitzpatrick.toml @@ -0,0 +1,68 @@ +# Regression case: GPEC's del_s Riccati layer-width solver against Fitzpatrick, "Tearing Mode +# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3 (p. 6-10), over (Q_hat, P_hat). +# Hand-checked at eight points against the published contour bands, e.g. (Q,P) = (4.0, 0.5) -> +# Re 2.008 against the 1.9-2.1 contours. The golden pins GPEC's output at that validated state on +# a provisional tolerance, so any re-pin must be re-checked against the figures. +# Pure ODE, no equilibrium or kinetic input: sub-second and fast-tier eligible. Complements +# slayer_delta_probe, which pins the dispersion solve Delta(Q). +[case] +name = "slayer_dels_fitzpatrick" +description = "del_s Riccati layer width vs Fitzpatrick 2023 figs 6.2/6.3 over (Q_hat, P_hat)" +kind = "computed" + +# The scan axes, pinned exactly so a silently reshaped grid is a failure rather than a re-gold. +[quantities.q_hat] +h5path = "fitzpatrick/Q_hat" +type = "real_vector" +extract = "all_real" +label = "grid Q_hat (= Q_*/D^4)" +noise_threshold = 0 +order = 10 +class = "topological" # exact input, not a converged output + +[quantities.p_hat] +h5path = "fitzpatrick/P_hat" +type = "real_vector" +extract = "all_real" +label = "grid P_hat (= P/D^6)" +noise_threshold = 0 +order = 11 +class = "topological" # exact input, not a converged output + +# tau is not stated in the book's figure captions; tau = 1 (table 5.1, low-field reactor) is the +# assumption this reproduction rests on, so it is pinned rather than left implicit. +[quantities.tau] +h5path = "fitzpatrick/tau" +type = "real_scalar" +extract = "value" +label = "tau (assumed; not given in the figure captions)" +noise_threshold = 0 +order = 12 +class = "topological" # exact input, not a converged output + +[quantities.dels_db_re] +h5path = "fitzpatrick/dels_db_re" +type = "real_vector" +extract = "all_real" +label = "Re(delta_s/d_beta) — Fitzpatrick fig 6.2" +noise_threshold = 1e-10 +order = 20 +class = "physics_converged" + +[quantities.dels_db_im] +h5path = "fitzpatrick/dels_db_im" +type = "real_vector" +extract = "all_real" +label = "Im(delta_s/d_beta) — Fitzpatrick fig 6.3" +noise_threshold = 1e-10 +order = 21 +class = "physics_converged" + +[quantities.runtime] +h5path = "" +type = "runtime" +extract = "value" +label = "Runtime (s)" +noise_threshold = 0.0 +order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/slayer_delta_probe.toml b/regression-harness/cases/slayer_delta_probe.toml new file mode 100644 index 000000000..a42bb4c76 --- /dev/null +++ b/regression-harness/cases/slayer_delta_probe.toml @@ -0,0 +1,55 @@ +# Regression case: fixed-Q probe of the SLAYER inner-layer dispersion Δ(Q). +# Evaluates Δ(Q) on a fixed 4×4 grid over Re(Q), Im(Q) ∈ [-10, 10] using the DIII-D-like +# SLAYER deck's own 2/1-surface layer parameters, quoted verbatim in the runner's computed +# script. Fixed Q values are refinement-stable, unlike the AMR scan samples in gpec.h5, so +# this pins the dispersion curve itself; the parameter chain producing those layer inputs +# is pinned separately by diiid_slayer_n1. Seconds to run — fast-tier eligible. No golden file yet. +[case] +name = "slayer_delta_probe" +description = "SLAYER inner-layer Δ(Q) on a fixed 4×4 Q grid, DIII-D-like 2/1 layer parameters" +kind = "computed" + +[quantities.q_re] +h5path = "slayer_probe/Q_re" +type = "real_vector" +extract = "all_real" +label = "probe grid Re(Q)" +noise_threshold = 0 +order = 10 +class = "topological" # exact input, not a converged output + +[quantities.q_im] +h5path = "slayer_probe/Q_im" +type = "real_vector" +extract = "all_real" +label = "probe grid Im(Q)" +noise_threshold = 0 +order = 11 +class = "topological" # exact input, not a converged output + +[quantities.delta_re] +h5path = "slayer_probe/Delta_re" +type = "real_vector" +extract = "all_real" +label = "SLAYER inner-layer Re Δ(Q) on the fixed grid" +noise_threshold = 1e-10 +order = 20 +class = "physics_converged" + +[quantities.delta_im] +h5path = "slayer_probe/Delta_im" +type = "real_vector" +extract = "all_real" +label = "SLAYER inner-layer Im Δ(Q) on the fixed grid" +noise_threshold = 1e-10 +order = 21 +class = "physics_converged" + +[quantities.runtime] +h5path = "" +type = "runtime" +extract = "value" +label = "Runtime (s)" +noise_threshold = 0.0 +order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_calculated.toml b/regression-harness/cases/solovev_kinetic_calculated.toml index d793afe9f..80d5b322e 100644 --- a/regression-harness/cases/solovev_kinetic_calculated.toml +++ b/regression-harness/cases/solovev_kinetic_calculated.toml @@ -16,6 +16,7 @@ type = "complex_vector" extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -23,6 +24,7 @@ type = "complex_vector" extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -30,6 +32,7 @@ type = "complex_vector" extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -37,6 +40,7 @@ type = "complex_vector" extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.et_all] @@ -45,6 +49,7 @@ type = "complex_vector" extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 +class = "physics_converged" # Integration [quantities.nstep] @@ -53,6 +58,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (saved)" noise_threshold = 0 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -60,6 +66,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (total)" noise_threshold = 0 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -68,6 +75,7 @@ type = "real_scalar" extract = "value" label = "q0" noise_threshold = 1e-12 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -75,6 +83,7 @@ type = "real_scalar" extract = "value" label = "q95" noise_threshold = 1e-12 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -83,6 +92,7 @@ type = "int_scalar" extract = "value" label = "# singular surfaces" noise_threshold = 0 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -90,6 +100,7 @@ type = "real_vector" extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -97,6 +108,7 @@ type = "real_vector" extract = "all_real" label = "singular q values" noise_threshold = 1e-8 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -105,6 +117,7 @@ type = "int_scalar" extract = "value" label = "mpert" noise_threshold = 0 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -112,6 +125,7 @@ type = "int_scalar" extract = "value" label = "npert" noise_threshold = 0 +class = "topological" # Runtime (special: not from H5) [quantities.runtime] @@ -120,3 +134,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" noise_threshold = 0.0 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_multiion.toml b/regression-harness/cases/solovev_kinetic_multiion.toml index faf23acfc..fd3285bf5 100644 --- a/regression-harness/cases/solovev_kinetic_multiion.toml +++ b/regression-harness/cases/solovev_kinetic_multiion.toml @@ -16,6 +16,7 @@ extract = "value" label = "NTV total (D+T+imp+e) [N·m]" noise_threshold = 1e-8 order = 10 +class = "physics_converged" [quantities.ntv_D] h5path = "KineticForces/PerSpecies/ion_z1_m2/fgar/total_torque" @@ -24,6 +25,7 @@ extract = "value" label = "NTV Deuterium [N·m]" noise_threshold = 1e-8 order = 11 +class = "physics_converged" [quantities.ntv_T] h5path = "KineticForces/PerSpecies/ion_z1_m3/fgar/total_torque" @@ -32,6 +34,7 @@ extract = "value" label = "NTV Tritium [N·m]" noise_threshold = 1e-8 order = 12 +class = "physics_converged" [quantities.ntv_electron] h5path = "KineticForces/PerSpecies/electron/fgar/total_torque" @@ -40,6 +43,7 @@ extract = "value" label = "NTV electron [N·m]" noise_threshold = 1e-8 order = 14 +class = "physics_converged" # Stability anchors — confirm the upstream FFS/PE stages feeding the NTV diagnostic. [quantities.et_real] @@ -49,6 +53,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.msing] h5path = "SingularSurfaces/rational_count" @@ -57,6 +62,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 21 +class = "topological" [quantities.runtime] h5path = "" @@ -65,3 +71,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_ntv.toml b/regression-harness/cases/solovev_kinetic_ntv.toml index a9a4d5324..6d59556a9 100644 --- a/regression-harness/cases/solovev_kinetic_ntv.toml +++ b/regression-harness/cases/solovev_kinetic_ntv.toml @@ -18,6 +18,7 @@ extract = "all_real" label = "NTV torque fgar [Re, Im]" noise_threshold = 1e-8 order = 10 +class = "physics_converged" [quantities.ntv_psi_nsteps] h5path = "KineticForces/fgar/psi_nsteps" @@ -26,6 +27,7 @@ extract = "value" label = "NTV ψ quadrature evaluations" noise_threshold = 0 order = 11 +class = "diagnostic" # adaptive step count: describes the numerics, never gates # Stability anchors — confirm the upstream FFS/PE stages feeding the NTV diagnostic. [quantities.et_real] @@ -35,6 +37,7 @@ extract = "real_first" label = "root-area-weighted total energy Re(et[1])" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.msing] h5path = "SingularSurfaces/rational_count" @@ -43,6 +46,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 21 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -51,6 +55,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 22 +class = "physics_converged" [quantities.q0] h5path = "Equilibrium/q_axis" @@ -59,6 +64,7 @@ extract = "value" label = "q0" noise_threshold = 0 order = 23 +class = "equilibrium_scalar" # Runtime (special: not from H5) [quantities.runtime] @@ -67,3 +73,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" noise_threshold = 0.0 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_nuzero.toml b/regression-harness/cases/solovev_kinetic_nuzero.toml index deb449473..75958f31d 100644 --- a/regression-harness/cases/solovev_kinetic_nuzero.toml +++ b/regression-harness/cases/solovev_kinetic_nuzero.toml @@ -20,6 +20,7 @@ type = "complex_vector" extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -27,6 +28,7 @@ type = "complex_vector" extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -34,6 +36,7 @@ type = "complex_vector" extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -41,6 +44,7 @@ type = "complex_vector" extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.et_all] @@ -49,6 +53,7 @@ type = "complex_vector" extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 +class = "physics_converged" # Integration [quantities.nstep] @@ -57,6 +62,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (saved)" noise_threshold = 0 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -64,6 +70,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (total)" noise_threshold = 0 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -72,6 +79,7 @@ type = "real_scalar" extract = "value" label = "q0" noise_threshold = 1e-12 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -79,6 +87,7 @@ type = "real_scalar" extract = "value" label = "q95" noise_threshold = 1e-12 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -87,6 +96,7 @@ type = "int_scalar" extract = "value" label = "# singular surfaces" noise_threshold = 0 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -94,6 +104,7 @@ type = "real_vector" extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -101,6 +112,7 @@ type = "real_vector" extract = "all_real" label = "singular q values" noise_threshold = 1e-8 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -109,6 +121,7 @@ type = "int_scalar" extract = "value" label = "mpert" noise_threshold = 0 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -116,6 +129,7 @@ type = "int_scalar" extract = "value" label = "npert" noise_threshold = 0 +class = "topological" # Runtime (special: not from H5) [quantities.runtime] @@ -124,3 +138,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" noise_threshold = 0.0 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_multi_n.toml b/regression-harness/cases/solovev_multi_n.toml index 2a6692540..e54bd353a 100644 --- a/regression-harness/cases/solovev_multi_n.toml +++ b/regression-harness/cases/solovev_multi_n.toml @@ -15,6 +15,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -23,6 +24,7 @@ extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.vacuum_eigenvalue] h5path = "ForceFreeStates/FreeBoundaryStability/vacuum_eigenvalue" @@ -31,6 +33,7 @@ extract = "value" label = "vacuum matrix min eigenvalue" noise_threshold = 1e-10 order = 14 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.ep_all] @@ -40,6 +43,7 @@ extract = "all_complex" label = "plasma energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ev_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -48,6 +52,7 @@ extract = "all_complex" label = "vacuum energy (all)" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -56,6 +61,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 22 +class = "physics_converged" # Integration [quantities.nstep] @@ -65,6 +71,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -73,6 +80,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 31 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -82,6 +90,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 40 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -90,6 +99,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 41 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -99,6 +109,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 50 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -107,6 +118,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 51 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -116,6 +128,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 60 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -124,6 +137,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 61 +class = "topological" # Profiles (checksums) [quantities.q_profile] @@ -133,6 +147,7 @@ extract = "checksum" label = "q profile (checksum)" noise_threshold = 0 order = 70 +class = "physics_converged" # Runtime [quantities.runtime] @@ -142,3 +157,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_n1.toml b/regression-harness/cases/solovev_n1.toml index d7f4224e5..758092343 100644 --- a/regression-harness/cases/solovev_n1.toml +++ b/regression-harness/cases/solovev_n1.toml @@ -15,6 +15,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -23,6 +24,7 @@ extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -31,6 +33,7 @@ extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 order = 12 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -39,6 +42,7 @@ extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.vacuum_eigenvalue] h5path = "ForceFreeStates/FreeBoundaryStability/vacuum_eigenvalue" @@ -47,6 +51,7 @@ extract = "value" label = "vacuum matrix min eigenvalue" noise_threshold = 1e-10 order = 14 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.ep_all] @@ -56,6 +61,7 @@ extract = "all_complex" label = "plasma energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ev_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -64,6 +70,7 @@ extract = "all_complex" label = "vacuum energy (all)" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -72,6 +79,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 22 +class = "physics_converged" # Integration [quantities.nstep] @@ -81,6 +89,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -89,6 +98,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 31 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -98,6 +108,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 40 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -106,6 +117,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 41 +class = "equilibrium_scalar" [quantities.betat] h5path = "Equilibrium/beta_t" @@ -114,6 +126,7 @@ extract = "value" label = "beta_t" noise_threshold = 1e-12 order = 42 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -122,6 +135,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 43 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -131,6 +145,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 50 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -139,6 +154,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 51 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -147,6 +163,7 @@ extract = "all_real" label = "singular q values" noise_threshold = 1e-8 order = 52 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -156,6 +173,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 60 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -164,6 +182,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 61 +class = "topological" # Profiles (checksums for fast comparison of large arrays) [quantities.q_profile] @@ -173,6 +192,7 @@ extract = "checksum" label = "q profile (checksum)" noise_threshold = 0 order = 70 +class = "physics_converged" [quantities.pressure_profile] h5path = "Equilibrium/Profiles/mu0p" @@ -181,6 +201,7 @@ extract = "checksum" label = "pressure profile (checksum)" noise_threshold = 0 order = 71 +class = "physics_converged" # Bitwise reproducibility guard for the ideal asymptotic coefficients (issue: kinetic # runs used to dump uninitialized memory here; ideal values must stay byte-stable). @@ -191,6 +212,7 @@ extract = "checksum" label = "ca_left (checksum)" noise_threshold = 0 order = 72 +class = "physics_converged" # Runtime (special: not from H5) [quantities.runtime] @@ -200,3 +222,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/golden/slayer_dels_fitzpatrick.toml b/regression-harness/golden/slayer_dels_fitzpatrick.toml new file mode 100644 index 000000000..be3bc29a2 --- /dev/null +++ b/regression-harness/golden/slayer_dels_fitzpatrick.toml @@ -0,0 +1,51 @@ +[meta] +blas_threads = 8 +case = "slayer_dels_fitzpatrick" +commit = "67a2c40cb" +generated_at = "2026-09-23" +golden_version = 2 +julia_version = "1.11.6" +manifest_sha = "7e5c34ad1722fa6addf94f7387b972fa8ac17ff86166793819b158496d495132" +nthreads = 8 +os_arch = "arm64-apple-darwin24.0.0" +reason = "Pin del_s/d_beta over the Fitzpatrick figs 6.2/6.3 (Q_hat, P_hat) grid, with the scan axes and tau exact; values unchanged from the figure-validated state" + +[values.dels_db_im] +atol = 0.0 +class = "physics_converged" +rtol = 1.0e-6 +tolerance_basis = "class-default (provisional)" +value_text = "[-0.03903669609227284,-0.7097932061530505,-1.0865009301031268,-1.5690999603536648,-2.2242340056733148,-0.011118192607611124,-0.35424388489008746,-0.776484631542084,-1.416939644030615,-2.2250603072320665,-0.008402000357094586,-0.23928032159546678,-0.5666784684880879,-1.241924594739638,-2.1750391797697133,-0.006297613829213364,-0.1657149360631757,-0.3716832232708168,-0.9092532037784165,-1.9732264855758332,-0.004682490543220957,-0.11916296976842101,-0.2502562086693129,-0.578522722631012,-1.4658508790642786]" +value_type = "json_array" + +[values.dels_db_re] +atol = 0.0 +class = "physics_converged" +rtol = 1.0e-6 +tolerance_basis = "class-default (provisional)" +value_text = "[0.46252746222290736,0.7691732891864712,1.0821020639186198,1.5537655127977916,2.2130417549691974,0.9838377906344368,0.9426353501706125,1.017567466663986,1.35072260354625,2.0080394342246684,1.1484429925403958,1.1069495858506195,1.0754748077370269,1.2328881423647,1.8273926389542299,1.336215433387759,1.3118923623802783,1.259193827055807,1.2120466712964955,1.540405108811276,1.5494641775774003,1.5373751336175125,1.5040680534472435,1.4084024975856595,1.350103015944749]" +value_type = "json_array" + +[values.p_hat] +atol = 0.0 +class = "topological" +rtol = 0.0 +tolerance_basis = "class-default (provisional)" +value_text = "[0.02,0.02,0.02,0.02,0.02,0.5,0.5,0.5,0.5,0.5,1.0,1.0,1.0,1.0,1.0,2.0,2.0,2.0,2.0,2.0,4.0,4.0,4.0,4.0,4.0]" +value_type = "json_array" + +[values.q_hat] +atol = 0.0 +class = "topological" +rtol = 0.0 +tolerance_basis = "class-default (provisional)" +value_text = "[0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0]" +value_type = "json_array" + +[values.tau] +atol = 0.0 +class = "topological" +rtol = 0.0 +tolerance_basis = "class-default (provisional)" +value = 1.0 +value_type = "real" diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 651a3df8d..f6f890e40 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -6,6 +6,7 @@ const HARNESS_DIR = @__DIR__ const REPO_ROOT = abspath(joinpath(HARNESS_DIR, "..")) const DEFAULT_DB_PATH = joinpath(HARNESS_DIR, ".regress_cache.sqlite") const CASES_DIR = joinpath(HARNESS_DIR, "cases") +const GOLDEN_DIR = joinpath(HARNESS_DIR, "golden") include("src/types.jl") include("src/env.jl") @@ -15,6 +16,7 @@ include("src/utils.jl") include("src/extractor.jl") include("src/runner.jl") include("src/reporter.jl") +include("src/golden.jl") function parse_args(args) cases = String[] @@ -30,6 +32,10 @@ function parse_args(args) no_pin_manifest = false allow_env_mismatch = false fail_on_change = false + check = false + update_golden = false + accept_exceeding = false + reason = nothing help = false i = 1 @@ -59,6 +65,18 @@ function parse_args(args) elseif arg == "--fail-on-change" fail_on_change = true i += 1 + elseif arg == "--check" + check = true + i += 1 + elseif arg == "--update-golden" + update_golden = true + i += 1 + elseif arg == "--accept-exceeding" + accept_exceeding = true + i += 1 + elseif arg == "--reason" && i < length(args) + reason = args[i+1] + i += 2 elseif arg == "--cases" && i < length(args) cases = split(args[i+1], ",") |> collect .|> strip i += 2 @@ -83,7 +101,7 @@ function parse_args(args) end return CLIOptions(cases, refs, ref_range, force, list_cases, show_qty, show_case, db_path, verbose, - no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, help) + no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, check, update_golden, accept_exceeding, reason, help) end const HELP_TEXT = """ @@ -117,15 +135,34 @@ Options: re-running them --fail-on-change Exit non-zero if any tracked quantity changed (for CI use; a failed run always exits non-zero regardless) + --check Run the working tree and compare against the committed golden values + (regression-harness/golden/.toml). No CI workflow runs it yet. + --update-golden Regenerate the golden values from a fresh run. Requires --cases and + --reason, and refuses a working tree with uncommitted changes or with + untracked files under examples/, src/ or regression-harness/cases/. + --accept-exceeding With --update-golden, allow a re-pin in which a gating quantity moved + outside its old tolerance (refused otherwise). Each such move's old + value and deviation is recorded under [meta.exceeded] in the golden file. + --reason "..." Why the goldens changed. Mandatory with --update-golden: a golden + change is a claim about physics that a reviewer has to evaluate. --help Print this help message Environment: GPEC_REGRESS_THREADS Threads for GPEC subprocesses (default "auto" = all cores); the actual count is recorded in each run's env fingerprint +Golden values: + A golden file pins each quantity together with the tolerance it must be reproduced within + and the evidence behind that tolerance. Tolerances must come from measured convergence + plateau drift and cross-platform spread; until measured they are provisional class defaults + and say so. A quantity's class is declared in its case file (`class = ...`) or inferred, + never set in the golden file. A failing --check is fixed by explaining the physics or + fixing the regression, not by loosening rtol. A re-pin that moves a value beyond its old + tolerance drops the recorded evidence and marks the entry provisional. + Exit status: 0 all runs completed (and, with --fail-on-change, nothing changed) - 1 a run failed, or a quantity changed under --fail-on-change + 1 a run failed, a quantity changed under --fail-on-change, or a golden check failed Examples: # Compare two refs @@ -150,9 +187,17 @@ Examples: # Compare current working tree against develop julia --project=regression-harness regression-harness/regress.jl \\ --cases solovev_n1 --refs develop,local + + # Gate the working tree against the committed golden values + julia --project=regression-harness regression-harness/regress.jl --check --cases solovev_n1 + + # Regenerate goldens after an intended physics change + julia --project=regression-harness regression-harness/regress.jl \\ + --update-golden --cases solovev_n1 --reason "two-point b1 fix changes resonant field" """ function main(args=ARGS) + # Kept out of CLIOptions so this flag needs no change to the shared option struct. opts = parse_args(args) if opts.help @@ -189,14 +234,35 @@ function main(args=ARGS) [all_cases[name] for name in opts.cases if haskey(all_cases, name) || error("Unknown case: $name")] end - # Resolve refs + golden_mode = opts.check || opts.update_golden + if opts.check && opts.update_golden + error("--check and --update-golden are mutually exclusive: one asserts the goldens, the other rewrites them") + end + if opts.update_golden && (opts.reason === nothing || isempty(strip(opts.reason))) + error("--update-golden requires --reason \"...\": a golden change is a claim about physics, " * + "and the reviewer needs to know what changed and why") + end + if opts.accept_exceeding && !opts.update_golden + error("--accept-exceeding only applies to --update-golden") + end + if opts.update_golden && isempty(opts.cases) + error("--update-golden requires --cases: goldens are regenerated one deliberate case at a time") + end + + # Resolve refs. Golden modes judge the working tree, so they default to it rather than + # requiring the caller to spell out --refs local. resolved_refs = if opts.ref_range !== nothing expand_ref_range(opts.ref_range, REPO_ROOT) elseif !isempty(opts.refs) [resolve_ref(ref, REPO_ROOT) for ref in opts.refs] + elseif golden_mode + [resolve_ref(LOCAL_REF, REPO_ROOT)] else error("Must specify --refs or --ref-range") end + if golden_mode && length(resolved_refs) != 1 + error("Golden modes compare one ref against the committed values; got $(length(resolved_refs)) refs") + end if isempty(resolved_refs) error("No commits resolved from the given refs") @@ -220,6 +286,11 @@ function main(args=ARGS) n_failed = 0 n_changed = 0 + n_golden_fail = 0 + n_golden_crashed = 0 + n_untracked = 0 + n_checked = 0 + n_no_golden = 0 # Run every case against each ref, grouped by ref so cases sharing a commit share # one worktree (and one Pkg.instantiate/precompile) instead of paying for it per case. @@ -236,20 +307,51 @@ function main(args=ARGS) println("Case: $(case_spec.name) — $(case_spec.description)") println("="^64) - summary = if length(resolved_refs) == 2 - report_two_ref_comparison(db, case_spec, - resolved_refs[1], resolved_refs[2]) + if opts.update_golden + update_golden_from_run(db, case_spec, resolved_refs[1].commit_hash, + String(opts.reason), REPO_ROOT; accept_exceeding=opts.accept_exceeding) + elseif opts.check + summary = report_golden_check(db, case_spec, resolved_refs[1].commit_hash) + n_golden_fail += summary.n_fail + n_golden_crashed += summary.n_run_failed + n_untracked += summary.n_untracked + summary.n_pass + summary.n_fail > 0 ? (n_checked += 1) : (n_no_golden += 1) else - report_multi_ref(db, case_spec, resolved_refs) + summary = if length(resolved_refs) == 2 + report_two_ref_comparison(db, case_spec, + resolved_refs[1], resolved_refs[2]) + else + report_multi_ref(db, case_spec, resolved_refs) + end + n_failed += summary.n_failed + n_changed += summary.n_changed end - n_failed += summary.n_failed - n_changed += summary.n_changed end if n_failed > 0 @error "$n_failed run(s) failed — see the reports above" exit(1) end + if n_golden_crashed > 0 + @error "$n_golden_crashed case(s) crashed before producing results — see the reports above (a crash, not a tolerance failure)" + exit(1) + end + if n_golden_fail > 0 + @error "$n_golden_fail quantity/quantities are outside their golden tolerance" + exit(1) + end + if opts.check && n_checked == 0 + # A green gate that checked nothing is worse than a red one: a deleted or + # typo-named golden file must not be indistinguishable from a passing check. + @error "--check ran $n_no_golden case(s) but none has a gating golden entry — nothing was actually gated" + exit(1) + end + if opts.check && n_no_golden > 0 + @warn "$n_no_golden of $(n_no_golden + n_checked) requested case(s) have no gating golden entry and were not gated" + end + if n_untracked > 0 + @info "$n_untracked tracked quantity/quantities have no golden value yet (not gating)" + end if opts.fail_on_change && n_changed > 0 @error "$n_changed quantity/quantities changed (--fail-on-change)" exit(1) diff --git a/regression-harness/src/config.jl b/regression-harness/src/config.jl index 4bee9d408..a0c9b660f 100644 --- a/regression-harness/src/config.jl +++ b/regression-harness/src/config.jl @@ -14,6 +14,9 @@ function load_case(filepath::String)::CaseSpec quantities = QuantitySpec[] if haskey(data, "quantities") for (qty_name, qty_data) in data["quantities"] + # Every quantity declares its golden tolerance class; there is no inference from its name. + haskey(qty_data, "class") || error("$filepath: quantity '$qty_name' has no `class`; declare one of $(join(TOLERANCE_CLASSES, ", "))") + qty_data["class"] in TOLERANCE_CLASSES || error("$filepath: quantity '$qty_name' has unknown class '$(qty_data["class"])'") push!( quantities, QuantitySpec( @@ -23,7 +26,8 @@ function load_case(filepath::String)::CaseSpec qty_data["extract"], get(qty_data, "label", qty_name), get(qty_data, "noise_threshold", 1e-10), - get(qty_data, "order", 1000) + get(qty_data, "order", 1000), + qty_data["class"] ) ) end diff --git a/regression-harness/src/database.jl b/regression-harness/src/database.jl index c9a06b303..3a5cecb77 100644 --- a/regression-harness/src/database.jl +++ b/regression-harness/src/database.jl @@ -50,7 +50,9 @@ added by a later schema migration (NULL on every pre-existing row) need both cas """ _column(x, default) = (x === nothing || x === missing) ? default : x -"""Materialize SQLite query results as a Vector of NamedTuples.""" +""" +Materialize SQLite query results as a Vector of NamedTuples. +""" function query_rows(db::SQLite.DB, sql::String, params=()) result = DBInterface.execute(db, sql, params) ct = Tables.columntable(result) @@ -70,7 +72,9 @@ const ENV_COLUMNS = [ ("pinned", "INTEGER") ] -"""Add any `runs` columns missing from a database created by an earlier harness version.""" +""" +Add any `runs` columns missing from a database created by an earlier harness version. +""" function migrate_schema!(db::SQLite.DB) existing = Set(String[]) for row in query_rows(db, "PRAGMA table_info(runs)") @@ -84,7 +88,8 @@ end function open_database(path::String)::SQLite.DB db = SQLite.DB(path) - DBInterface.execute(db, "PRAGMA journal_mode=WAL") + # journal_mode returns a row; close the query so no statement is left in progress to block a later transaction. + DBInterface.close!(DBInterface.execute(db, "PRAGMA journal_mode=WAL")) DBInterface.execute(db, "PRAGMA foreign_keys=ON") for stmt in split(SCHEMA_SQL, ";") s = strip(stmt) @@ -107,10 +112,10 @@ environment. Rows predating fingerprinting hold NULL and therefore never match entries most likely to be misleading are exactly the ones that get re-run. """ function is_cached(db::SQLite.DB, commit_hash::String, case_name::String; - expected_key::Union{String,Nothing}=nothing)::Bool + expected_key::Union{String,Nothing}=nothing)::Bool if expected_key === nothing rows = query_rows(db, "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1", - (commit_hash, case_name)) + (commit_hash, case_name)) return !isempty(rows) end rows = query_rows(db, @@ -125,7 +130,7 @@ fingerprinting. Used to explain *why* a cached result was rejected. """ function cached_env_key(db::SQLite.DB, commit_hash::String, case_name::String)::Union{String,Nothing} rows = query_rows(db, "SELECT env_key FROM runs WHERE commit_hash = ? AND case_name = ?", - (commit_hash, case_name)) + (commit_hash, case_name)) isempty(rows) && return nothing key = _column(first(rows).env_key, nothing) key === nothing && return nothing @@ -135,15 +140,15 @@ end function delete_cached(db::SQLite.DB, commit_hash::String, case_name::String) # ON DELETE CASCADE handles quantities cleanup automatically DBInterface.execute(db, "DELETE FROM runs WHERE commit_hash = ? AND case_name = ?", - (commit_hash, case_name)) + (commit_hash, case_name)) end function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::AbstractString, - commit_date::AbstractString, commit_msg::AbstractString, - case_name::AbstractString, runtime_s::Float64, - extracted::Vector{ExtractedQuantity}; - success::Bool=true, error_msg::AbstractString="", - fingerprint::EnvFingerprint=UNKNOWN_ENV) + commit_date::AbstractString, commit_msg::AbstractString, + case_name::AbstractString, runtime_s::Float64, + extracted::Vector{ExtractedQuantity}; + success::Bool=true, error_msg::AbstractString="", + fingerprint::EnvFingerprint=UNKNOWN_ENV) ran_at = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS") SQLite.transaction(db) do @@ -155,10 +160,10 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs env_key, julia_version, os_arch, manifest_sha, nthreads, blas_threads, pinned) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (String(commit_hash), String(commit_short), String(commit_date), String(commit_msg), - String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg), - env_key(fingerprint), fingerprint.julia_version, fingerprint.os_arch, - fingerprint.manifest_sha, fingerprint.nthreads, fingerprint.blas_threads, - fingerprint.pinned ? 1 : 0)) + String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg), + env_key(fingerprint), fingerprint.julia_version, fingerprint.os_arch, + fingerprint.manifest_sha, fingerprint.nthreads, fingerprint.blas_threads, + fingerprint.pinned ? 1 : 0)) run_id = SQLite.last_insert_rowid(db) @@ -168,14 +173,14 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs (run_id, qty_name, qty_label, value_real, value_int, value_text, value_type, noise_threshold) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", (run_id, eq.name, eq.label, eq.value_real, eq.value_int, eq.value_text, - eq.value_type, eq.noise_threshold)) + eq.value_type, eq.noise_threshold)) end end end function store_failed_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::AbstractString, - commit_date::AbstractString, commit_msg::AbstractString, - case_name::AbstractString, error_msg::AbstractString) + commit_date::AbstractString, commit_msg::AbstractString, + case_name::AbstractString, error_msg::AbstractString) ran_at = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS") delete_cached(db, String(commit_hash), String(case_name)) DBInterface.execute(db, @@ -183,14 +188,14 @@ function store_failed_run(db::SQLite.DB, commit_hash::AbstractString, commit_sho (commit_hash, commit_short, commit_date, commit_msg, case_name, ran_at, runtime_s, success, error_msg) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", (String(commit_hash), String(commit_short), String(commit_date), String(commit_msg), - String(case_name), ran_at, 0.0, 0, String(error_msg))) + String(case_name), ran_at, 0.0, 0, String(error_msg))) end """ Get all quantities for a (commit, case) pair. Returns Dict{qty_name => NamedTuple}. """ function get_quantities(db::SQLite.DB, commit_hash::String, case_name::String) - results = Dict{String, NamedTuple}() + results = Dict{String,NamedTuple}() rows = query_rows(db, """SELECT q.qty_name, q.qty_label, q.value_real, q.value_int, q.value_text, q.value_type, q.noise_threshold @@ -202,12 +207,12 @@ function get_quantities(db::SQLite.DB, commit_hash::String, case_name::String) for row in rows name = something(row.qty_name, "") results[name] = ( - label = something(row.qty_label, name), - value_real = row.value_real, - value_int = row.value_int, - value_text = row.value_text, - value_type = something(row.value_type, "missing"), - noise_threshold = something(row.noise_threshold, 0.0), + label=something(row.qty_label, name), + value_real=row.value_real, + value_int=row.value_int, + value_text=row.value_text, + value_type=something(row.value_type, "missing"), + noise_threshold=something(row.noise_threshold, 0.0) ) end return results @@ -233,13 +238,13 @@ function get_run_info(db::SQLite.DB, commit_hash::String, case_name::String) _column(row.pinned, 0) == 1 ) return ( - commit_short = something(row.commit_short, ""), - commit_date = something(row.commit_date, ""), - commit_msg = something(row.commit_msg, ""), - runtime_s = something(row.runtime_s, 0.0), - success = coalesce(row.success, 0) == 1, - error_msg = something(row.error_msg, ""), - fingerprint = fingerprint, + commit_short=something(row.commit_short, ""), + commit_date=something(row.commit_date, ""), + commit_msg=something(row.commit_msg, ""), + runtime_s=something(row.runtime_s, 0.0), + success=coalesce(row.success, 0) == 1, + error_msg=something(row.error_msg, ""), + fingerprint=fingerprint ) end diff --git a/regression-harness/src/extractor.jl b/regression-harness/src/extractor.jl index 33a328d7c..a4747158c 100644 --- a/regression-harness/src/extractor.jl +++ b/regression-harness/src/extractor.jl @@ -112,6 +112,19 @@ function apply_extraction(spec::QuantitySpec, raw)::ExtractedQuantity json_str = JSON.json(pairs; allownan=true) return ExtractedQuantity(name, label, nothing, nothing, json_str, "json_array", threshold) + elseif startswith(spec.extract, "toml_key:") + # Pin a deck's declared control value from the Input/ snapshot; an unset key pins as + # "" so starting or stopping to declare it is itself a change. + keypath = spec.extract[(length("toml_key:")+1):end] + table = TOML.parse(raw isa AbstractString ? raw : String(raw)) + node = table + for key in split(keypath, ".") + node = (node isa AbstractDict && haskey(node, key)) ? node[key] : nothing + node === nothing && break + end + token = node === nothing ? "" : string(node) + return ExtractedQuantity(name, label, nothing, nothing, token, "token", threshold) + elseif spec.extract == "checksum" bytes = reinterpret(UInt8, vec(collect(raw))) hash = bytes2hex(sha256(bytes)) @@ -132,6 +145,7 @@ function _json_element_diff(a, b)::Float64 (isnan(Float64(a)) && isnan(Float64(b))) && return 0.0 return abs(Float64(a) - Float64(b)) elseif a isa Vector && b isa Vector + length(a) == length(b) || return Inf return sqrt(sum(_json_element_diff(ai, bi)^2 for (ai, bi) in zip(a, b))) else return Inf @@ -218,7 +232,7 @@ function compare_values(q1::NamedTuple, q2::NamedTuple) status = max_diff <= threshold ? "OK" : "CHANGED" return (max_diff, rel_diff, status) - elseif vtype == "checksum" + elseif vtype == "checksum" || vtype == "token" t1 = q1.value_text t2 = q2.value_text if t1 === nothing || t2 === nothing diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl new file mode 100644 index 000000000..d43988f7d --- /dev/null +++ b/regression-harness/src/golden.jl @@ -0,0 +1,682 @@ +""" +Golden values: committed, reviewable reference numbers for a case. + +The `--refs` comparison answers "did this change?"; it cannot answer "is this right?", because +both sides come from the same working copy of the code and nothing is recorded in git. Golden +values close that gap: one TOML file per case, tracked alongside the source, holding the value +of every quantity plus the tolerance it must be reproduced within and the evidence behind that +tolerance. + +The tolerance policy is the load-bearing part. A tolerance is a claim about how much a quantity +may legitimately move, so it must come from measurement — the residual drift at a convergence +plateau and the spread across platforms — never be chosen to make a check pass. Until measured, an +entry carries a provisional class default and is stamped as such. `save_golden` and `load_golden` +refuse a tolerance below either recorded measurement, and a failing check is fixed by explaining +the physics or fixing the regression, not by widening the bound. +""" + +""" +How strictly a quantity must reproduce, and why. + + - `topological` — integer counts, mode numbers, and exact inputs such as echoed scan grids; + exact equality + - `equilibrium_scalar` — spline/quadrature outputs with no adaptive branching; very tight + - `physics_converged` — the quantities golden values exist for (δW, Δ′, torque, growth + rates); tolerance comes from the measured convergence plateau and platform spread + - `diagnostic` — ODE step counts, runtimes; recorded and reported, never gating, because they + describe the numerics rather than the physics + - `unconverged` — measured and found to have no plateau; tracked differentially, never pinned, + so that a known-unconverged quantity is visibly excluded instead of quietly given a wide bound + +A quantity's class comes only from the `class` key its case file must declare, never from the +golden file or the quantity's name, so demoting a gate to a non-gating class is a reviewed +case-file change and renaming a quantity cannot silently change its class. + +A gating entry's `atol` must be 0 unless its golden value contains an exact zero (a scalar equal to +0, or an array element whose every component is 0). Everywhere else the relative test is the whole +gate; an absolute floor is only for structural zeros, where the relative test is meaningless. +""" +const TOLERANCE_CLASSES = ("topological", "equilibrium_scalar", "physics_converged", "diagnostic", "unconverged") + +""" +Classes whose failure fails the run. `diagnostic` and `unconverged` are reported only. +""" +const GATING_CLASSES = ("topological", "equilibrium_scalar", "physics_converged") + +""" +Provisional per-class tolerances, used only when no measurement has been supplied. + +These are starting points, not results: a golden file written with them is stamped +`tolerance_basis = "class-default (provisional)"` so that an unmeasured pin is never mistaken +for a converged one. +""" +const CLASS_DEFAULT_TOLERANCE = Dict( + "topological" => (rtol=0.0, atol=0.0), + "equilibrium_scalar" => (rtol=1e-9, atol=0.0), + "physics_converged" => (rtol=1e-6, atol=0.0), + "diagnostic" => (rtol=Inf, atol=Inf), + "unconverged" => (rtol=Inf, atol=Inf) +) + +""" +One quantity's pinned value and the terms it is judged by. + +## Fields + + - `name` — quantity key, matching the case's `[quantities.*]` table + - `value_type` — "real", "integer", "json_array" (mirrors `ExtractedQuantity`) + - `value_real` / `value_int` / `value_text` — the pinned value in its stored form + - `rtol` / `atol` — pass when `|x - gold| <= atol + rtol*|gold|` + - `class` — one of `TOLERANCE_CLASSES` + - `tolerance_basis` — how the tolerance was arrived at: "measured" (requires a recorded + `plateau_drift` or `platform_spread`), "class-default (provisional)", or "provisional" (no + measurement, tolerance kept tighter than the class default after a re-pin dropped the evidence) + - `plateau_drift` — relative movement over the final refinement of the convergence scan; NaN + when not measured + - `platform_spread` — relative disagreement between reference platforms; NaN when not measured + - `converged_at` — the discretization the value was taken at, e.g. + "grid_type=ldp, mpsi=1024, mtheta=512" +""" +struct GoldenValue + name::String + value_type::String + value_real::Union{Float64,Nothing} + value_int::Union{Int,Nothing} + value_text::Union{String,Nothing} + rtol::Float64 + atol::Float64 + class::String + tolerance_basis::String + plateau_drift::Float64 + platform_spread::Float64 + converged_at::String +end + +is_gating(g::GoldenValue) = g.class in GATING_CLASSES + +_is_exact_zero(x) = x isa Number ? x == 0 : (x isa Vector && !isempty(x) && all(_is_exact_zero, x)) + +""" +True when a value holds an exact zero at the granularity `compare_to_golden` judges it: a zero +scalar, or an array element (a number, or a nested row or complex pair) that is zero throughout. +""" +function has_exact_zero(value_type, value_real, value_int, value_text) + value_type == "real" && return value_real !== nothing && value_real == 0.0 + value_type == "integer" && return value_int !== nothing && value_int == 0 + value_type == "json_array" && value_text !== nothing && return any(_is_exact_zero, JSON.parse(value_text; allownan=true)) + return false +end +has_exact_zero(g::GoldenValue) = has_exact_zero(g.value_type, g.value_real, g.value_int, g.value_text) + +""" +True when a tolerance is not backed by a recorded measurement, i.e. any basis other than "measured". +""" +is_provisional(g::GoldenValue) = g.tolerance_basis != "measured" + +""" +Runtimes are wall-clock and checksums have no notion of "close", so neither is ever pinned. +""" +is_pinnable(spec::QuantitySpec) = spec.type != "runtime" && spec.extract != "checksum" + +""" +Refuse an entry that would gate more quietly than its evidence allows. Run on every load as well as +every save, so a hand edit or a merge taking the wrong side is caught as surely as a bad write. +""" +function validate_golden_value(g::GoldenValue, context::AbstractString) + g.class in TOLERANCE_CLASSES || error("$context: quantity '$(g.name)' has unknown class '$(g.class)'") + if is_gating(g) + (isfinite(g.rtol) && g.rtol >= 0 && isfinite(g.atol) && g.atol >= 0) || + error("$context: gating quantity '$(g.name)' needs a finite, non-negative rtol and atol (got rtol=$(g.rtol), atol=$(g.atol))") + g.atol == 0 || has_exact_zero(g) || + error("$context: gating quantity '$(g.name)' has atol $(g.atol) but no exact zero in its value; atol is only for structural zeros") + for (label, m) in (("platform_spread", g.platform_spread), ("plateau_drift", g.plateau_drift)) + isfinite(m) && g.rtol < m && + error( + "$context: quantity '$(g.name)' has rtol $(g.rtol) below its measured $label $m. " * + "Widen it deliberately, with the measurement recorded, or reduce the $label before pinning." + ) + end + end + g.tolerance_basis == "measured" && !isfinite(g.plateau_drift) && !isfinite(g.platform_spread) && + error("$context: quantity '$(g.name)' claims a measured tolerance but records neither plateau_drift nor platform_spread") + return nothing +end + +""" +Provenance for a whole golden file: what produced these numbers and why they last changed. + +## Fields + + - `exceeded` — for each gating quantity whose re-pin fell outside its previous tolerance (allowed + only with `--accept-exceeding`), a table of `previous_version`, `previous` (the old value), + `deviation` (as `compare_to_golden` reports it; NaN on a type change), `previous_rtol` and + `previous_atol`; written as `[meta.exceeded.]` so the git diff shows each accepted move +""" +struct GoldenMeta + case::String + golden_version::Int + generated_at::String + commit::String + reason::String + julia_version::String + os_arch::String + manifest_sha::String + nthreads::Int + blas_threads::Int + exceeded::Dict{String,Any} +end + +GoldenMeta(case, golden_version, generated_at, commit, reason, julia_version, os_arch, manifest_sha, nthreads, blas_threads) = + GoldenMeta(case, golden_version, generated_at, commit, reason, julia_version, os_arch, manifest_sha, nthreads, blas_threads, Dict{String,Any}()) + +golden_path(case_name::AbstractString) = joinpath(GOLDEN_DIR, "$(case_name).toml") + +has_golden(case_name::AbstractString) = isfile(golden_path(case_name)) + +""" +Read a golden file. Returns `(meta, Dict{name => GoldenValue})`, or `nothing` when absent. +""" +function load_golden(case_name::AbstractString) + path = golden_path(case_name) + isfile(path) || return nothing + data = TOML.parsefile(path) + m = get(data, "meta", Dict{String,Any}()) + meta = GoldenMeta( + get(m, "case", String(case_name)), + Int(get(m, "golden_version", 0)), + get(m, "generated_at", ""), + get(m, "commit", ""), + get(m, "reason", ""), + get(m, "julia_version", ""), + get(m, "os_arch", ""), + get(m, "manifest_sha", ""), + Int(get(m, "nthreads", -1)), + Int(get(m, "blas_threads", -1)), + Dict{String,Any}(get(m, "exceeded", Dict{String,Any}())) + ) + values = Dict{String,GoldenValue}() + for (name, v) in get(data, "values", Dict{String,Any}()) + g = GoldenValue( + name, + get(v, "value_type", "real"), + haskey(v, "value") && v["value"] isa Real ? Float64(v["value"]) : nothing, + haskey(v, "value_int") ? Int(v["value_int"]) : nothing, + haskey(v, "value_text") ? String(v["value_text"]) : nothing, + Float64(get(v, "rtol", NaN)), + Float64(get(v, "atol", 0.0)), + get(v, "class", "physics_converged"), + get(v, "tolerance_basis", "unspecified"), + Float64(get(v, "plateau_drift", NaN)), + Float64(get(v, "platform_spread", NaN)), + get(v, "converged_at", "") + ) + validate_golden_value(g, "Golden file $path") + values[name] = g + end + return (meta=meta, values=values) +end + +""" +Write a golden file. + +Refuses any entry `validate_golden_value` rejects, and any entry with no value. Raising a +tolerance to cover a measured spread is a deliberate act the caller performs, never something this +function does behind the caller's back. +""" +function save_golden(meta::GoldenMeta, values::Dict{String,GoldenValue}) + for g in Base.values(values) + if g.value_real === nothing && g.value_int === nothing && g.value_text === nothing + error( + "Quantity '$(g.name)': refusing to write a golden entry with no value (the run " * + "produced NaN or nothing) — a valueless gating entry fails forever and regenerating " * + "reproduces it. Exclude the quantity or fix the extraction." + ) + end + validate_golden_value(g, "Refusing to write golden for $(meta.case)") + end + mkpath(GOLDEN_DIR) + out = Dict{String,Any}( + "meta" => Dict{String,Any}( + "case" => meta.case, + "golden_version" => meta.golden_version, + "generated_at" => meta.generated_at, + "commit" => meta.commit, + "reason" => meta.reason, + "julia_version" => meta.julia_version, + "os_arch" => meta.os_arch, + "manifest_sha" => meta.manifest_sha, + "nthreads" => meta.nthreads, + "blas_threads" => meta.blas_threads + ) + ) + isempty(meta.exceeded) || (out["meta"]["exceeded"] = meta.exceeded) + vals = Dict{String,Any}() + for (name, g) in values + entry = Dict{String,Any}( + "value_type" => g.value_type, + "class" => g.class, + "tolerance_basis" => g.tolerance_basis, + "rtol" => g.rtol, + "atol" => g.atol + ) + g.value_real !== nothing && (entry["value"] = g.value_real) + g.value_int !== nothing && (entry["value_int"] = g.value_int) + g.value_text !== nothing && (entry["value_text"] = g.value_text) + isfinite(g.plateau_drift) && (entry["plateau_drift"] = g.plateau_drift) + isfinite(g.platform_spread) && (entry["platform_spread"] = g.platform_spread) + isempty(g.converged_at) || (entry["converged_at"] = g.converged_at) + vals[name] = entry + end + out["values"] = vals + open(golden_path(meta.case), "w") do io + TOML.print(io, out; sorted=true) + end + return golden_path(meta.case) +end + +""" +Return a quantity's declared tolerance class, validated against `TOLERANCE_CLASSES`. + +There is no inference from the quantity's name or type: a missing class is an error, so that +every gate is an explicit, reviewed choice in the case file. +""" +function infer_class(spec::QuantitySpec)::String + isempty(spec.class) && error("Quantity '$(spec.name)': no `class` declared in its case file") + spec.class in TOLERANCE_CLASSES || error("Quantity '$(spec.name)': unknown class '$(spec.class)' in its case file") + return spec.class +end + +""" +Compare one extracted quantity against its golden value. + +Returns `(passed, deviation, detail)` where `deviation` is the relative deviation (absolute when +the golden value is zero) and `detail` is a human-readable reason on failure. Array quantities +pass only when every element is within tolerance; the reported deviation is the worst element. +""" +function compare_to_golden(q::NamedTuple, g::GoldenValue) + # Non-gating classes are recorded, never judged; gating ones are validated finite on load. + within = (x, gold) -> !is_gating(g) || abs(x - gold) <= g.atol + g.rtol * abs(gold) + + if q.value_type != g.value_type + return (false, NaN, "type changed: golden $(g.value_type), got $(q.value_type)") + end + + if g.value_type == "integer" + (q.value_int === nothing || g.value_int === nothing) && return (false, NaN, "missing integer value") + d = Float64(abs(q.value_int - g.value_int)) + passed = g.class == "topological" ? q.value_int == g.value_int : within(Float64(q.value_int), Float64(g.value_int)) + return (passed, g.value_int == 0 ? d : d / abs(g.value_int), passed ? "" : "$(g.value_int) → $(q.value_int)") + + elseif g.value_type == "real" + (q.value_real === nothing || g.value_real === nothing) && return (false, NaN, "missing value") + gold = g.value_real + got = q.value_real + d = abs(got - gold) + rel = gold == 0.0 ? d : d / abs(gold) + return (within(got, gold), rel, within(got, gold) ? "" : @sprintf("%.9g → %.9g", gold, got)) + + elseif g.value_type == "json_array" + (q.value_text === nothing || g.value_text === nothing) && return (false, NaN, "missing array") + got = JSON.parse(q.value_text; allownan=true) + gold = JSON.parse(g.value_text; allownan=true) + length(got) == length(gold) && return _compare_arrays(got, gold, g) + return (false, NaN, "length $(length(gold)) → $(length(got))") + + elseif g.value_type == "token" + # Discrete choice, so no tolerance; deviation 1.0 (not NaN) on mismatch sorts as a failure. + (q.value_text === nothing || g.value_text === nothing) && return (false, NaN, "missing token") + matched = q.value_text == g.value_text + return (matched, matched ? 0.0 : 1.0, matched ? "" : "$(g.value_text) → $(q.value_text)") + end + + return (false, NaN, "unsupported value type $(g.value_type)") +end + +""" +Worst-element comparison for array quantities, shared by the real and complex encodings. +""" +function _compare_arrays(got, gold, g::GoldenValue) + worst_rel = 0.0 + worst_idx = 0 + all_ok = true + for i in eachindex(gold) + a = _json_element_abs(gold[i]) + d = _json_element_diff(gold[i], got[i]) + # A NaN on one side counts as the worst possible element, so the detail points at it. + rel = isnan(d) ? Inf : (a == 0.0 ? d : d / a) + ok = !is_gating(g) || d <= g.atol + g.rtol * a + ok || (all_ok = false) + if rel > worst_rel || (worst_idx == 0 && !ok) + worst_rel = rel + worst_idx = i + end + end + detail = all_ok ? "" : "worst element $(worst_idx): rel $(@sprintf("%.3e", worst_rel))" + return (all_ok, worst_rel, detail) +end + +""" +Compare a case's fresh run against its golden file and print the report. + +Returns `(n_pass, n_fail, n_untracked, n_informational, n_run_failed)`. Only gating classes +can fail; `diagnostic` and `unconverged` quantities are shown with their deviation and marked +as informational, so a reader sees them move without the run failing on them. A run that +crashed outright is counted in `n_run_failed`, never in `n_fail`, so the caller can exit with +a crash message instead of a tolerance message. +""" +function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String) + golden = load_golden(case_spec.name) + println() + println("Golden Check: $(case_spec.name)") + if golden === nothing + println(" No golden file at $(golden_path(case_spec.name)) — nothing to check against.") + println(" Generate one with: regress --update-golden --cases $(case_spec.name) --reason \"...\"") + return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=0) + end + + quantities = get_quantities(db, commit_hash, case_spec.name) + info = get_run_info(db, commit_hash, case_spec.name) + if info !== nothing && !info.success + println(" RUN FAILED — nothing to compare (this is a crash, not a tolerance failure):") + println(" $(_short_err(info.error_msg))") + return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=1) + end + + rows = Vector{Vector{String}}() + n_pass = n_fail = n_untracked = n_informational = n_provisional = 0 + + for spec in case_spec.quantities + g = get(golden.values, spec.name, nothing) + if g === nothing + is_pinnable(spec) || continue + n_untracked += 1 + continue + end + q_raw = get(quantities, spec.name, nothing) + if q_raw === nothing + push!(rows, [spec.label, g.class, "MISSING", "—", "—", "—", "FAIL"]) + n_fail += 1 + continue + end + # The golden file's class must be the one the case declares, so a demotion cannot hide there. + declared = infer_class(spec) + if g.class != declared + push!(rows, [spec.label, g.class, "CLASS", "—", "—", "—", "** FAIL ** case declares $declared; regenerate goldens"]) + n_fail += 1 + continue + end + # SQLite NULLs surface as `missing`; normalize them to nothing as the update path does. + q = (label=q_raw.label, value_real=_column(q_raw.value_real, nothing), + value_int=_column(q_raw.value_int, nothing), value_text=_column(q_raw.value_text, nothing), + value_type=q_raw.value_type, noise_threshold=q_raw.noise_threshold) + passed, deviation, detail = compare_to_golden(q, g) + gating = is_gating(g) + status = if !gating + "info" + elseif passed + "ok" + else + "** FAIL **" + end + if !gating + n_informational += 1 + elseif passed + n_pass += 1 + else + n_fail += 1 + end + # A scalar with a zero golden value is judged absolutely, so its deviation is marked as such. + zero_gold = (g.value_type == "real" && g.value_real == 0.0) || (g.value_type == "integer" && g.value_int == 0) + dev = isnan(deviation) ? "—" : @sprintf("%.2e", deviation) * (zero_gold ? " (abs)" : "") + rtol = isfinite(g.rtol) ? @sprintf("%.1e", g.rtol) : "—" + atol = isfinite(g.atol) ? @sprintf("%.1e", g.atol) : "—" + basis = !gating ? "—" : is_provisional(g) ? "provisional" : "measured" + gating && is_provisional(g) && (n_provisional += 1) + push!(rows, [spec.label, g.class, dev, rtol, atol, basis, status * (isempty(detail) ? "" : " $detail")]) + end + + # An orphaned entry fails regardless of class: a renamed quantity must not silently drop its gate. + spec_names = Set(spec.name for spec in case_spec.quantities) + for name in sort(collect(keys(golden.values))) + name in spec_names && continue + push!(rows, [name, golden.values[name].class, "ORPHANED", "—", "—", "—", + "** FAIL ** no matching quantity in the case — renamed or removed? Regenerate goldens."]) + n_fail += 1 + end + + header = ["Quantity", "Class", "Deviation", "rtol", "atol", "Basis", "Status"] + widths = [length(h) for h in header] + for row in rows, i in eachindex(row) + widths[i] = max(widths[i], length(row[i])) + end + total_w = sum(widths) + 2 * (length(header) - 1) + + println("="^total_w) + println("Golden v$(golden.meta.golden_version), generated $(golden.meta.generated_at) @ $(golden.meta.commit)") + isempty(golden.meta.reason) || println("Reason: $(golden.meta.reason)") + println("Golden env: julia $(golden.meta.julia_version), $(golden.meta.os_arch), $(golden.meta.nthreads) thread/$(golden.meta.blas_threads) BLAS") + if info !== nothing && !isempty(info.fingerprint.julia_version) + println("This run: $(describe_env(info.fingerprint))") + info.fingerprint.os_arch == golden.meta.os_arch || + println(" NOTE: different platform from the one the goldens were measured on; deviations at or below the recorded platform_spread are expected.") + end + println("-"^total_w) + _print_row(header, widths) + println("-"^total_w) + for row in rows + _print_row(row, widths) + end + println("="^total_w) + parts = String[] + n_pass > 0 && push!(parts, "$n_pass pass") + n_fail > 0 && push!(parts, "$n_fail FAIL") + n_informational > 0 && push!(parts, "$n_informational informational") + n_untracked > 0 && push!(parts, "$n_untracked untracked") + println("Summary: ", join(parts, ", ")) + n_provisional > 0 && println( + " $n_provisional gating quantity/quantities are judged against provisional class-default tolerances, " * + "not a measured plateau drift or platform spread." + ) + println() + return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational, n_run_failed=0) +end + +""" +Directories, relative to the repository root, whose untracked files could change a golden run's result. +""" +const GOLDEN_SOURCE_DIRS = ("examples", "src", "regression-harness/cases") + +""" +Changes that the recorded commit would not reproduce: uncommitted changes to tracked files under +`repo_root`, ignoring `exclude_dir` (the goldens themselves), plus untracked files under `source_dirs`. +""" +function uncommitted_changes(repo_root::AbstractString, exclude_dir::AbstractString; source_dirs=GOLDEN_SOURCE_DIRS) + rel = relpath(exclude_dir, repo_root) + tracked = strip(read(`git -C $repo_root status --porcelain --untracked-files=no -- . ":(exclude)$rel"`, String)) + status = read(`git -C $repo_root status --porcelain --untracked-files=all -- $(collect(source_dirs))`, String) + untracked = filter(l -> startswith(l, "??"), split(status, '\n')) + return String(strip(join([tracked; untracked], '\n'))) +end + +""" +Gating entries of `existing` whose regenerated value in `values` falls outside the old tolerance, +as `name => Dict` tables ready for `GoldenMeta.exceeded`. A value-type change always counts. +""" +function exceeding_changes(existing::Dict{String,GoldenValue}, values::Dict{String,GoldenValue}, previous_version::Int) + exceeded = Dict{String,Any}() + for (name, new) in values + old = get(existing, name, nothing) + (old === nothing || !is_gating(old)) && continue + q = (value_real=new.value_real, value_int=new.value_int, value_text=new.value_text, value_type=new.value_type) + passed, dev, _ = compare_to_golden(q, old) + passed && continue + exceeded[name] = Dict{String,Any}("previous_version" => previous_version, + "previous" => something(old.value_real, old.value_int, old.value_text, ""), + "deviation" => dev, "previous_rtol" => old.rtol, "previous_atol" => old.atol) + end + return exceeded +end + +""" +Regenerate a case's golden file from a completed run, reporting what moved. + +Prints an old→new delta for every quantity whose value or class changed, flagging any move that +the old tolerance would have failed, so the diff a reviewer sees in git comes with the size and +significance of each move. Refuses a working tree with uncommitted changes outside the golden +directory, or untracked files under `GOLDEN_SOURCE_DIRS`, because the recorded commit is what a +reviewer uses to reproduce a disputed number. Refuses a re-pin in which any gating quantity leaves +its old tolerance unless `accept_exceeding`, which records each such move in `[meta.exceeded]`. +""" +function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String, + reason::String, repo_root::String; accept_exceeding::Bool=false) + info = get_run_info(db, commit_hash, case_spec.name) + (info === nothing || !info.success) && error("Cannot update goldens for '$(case_spec.name)': the run did not succeed") + if commit_hash == LOCAL_REF + dirty = uncommitted_changes(repo_root, GOLDEN_DIR) + isempty(dirty) || error( + "Cannot update goldens for '$(case_spec.name)': the working tree has uncommitted changes " * + "or untracked source files, so the recorded commit would not reproduce these values. " * + "Commit (or remove) them first.\n$dirty" + ) + end + + extracted = ExtractedQuantity[] + for (name, q) in get_quantities(db, commit_hash, case_spec.name) + push!( + extracted, + ExtractedQuantity(name, String(_column(q.label, name)), + _column(q.value_real, nothing), _column(q.value_int, nothing), + _column(q.value_text, nothing), q.value_type, q.noise_threshold) + ) + end + + previous = load_golden(case_spec.name) + existing = previous === nothing ? nothing : previous.values + values = build_golden_values(extracted, case_spec.quantities, existing) + + exceeded = existing === nothing ? Dict{String,Any}() : exceeding_changes(existing, values, previous.meta.golden_version) + fp = info.fingerprint + meta = GoldenMeta(case_spec.name, + previous === nothing ? 1 : previous.meta.golden_version + 1, + Dates.format(Dates.now(), "yyyy-mm-dd"), + strip(read(`git -C $repo_root rev-parse --short $(commit_hash == LOCAL_REF ? "HEAD" : commit_hash)`, String)), + reason, fp.julia_version, fp.os_arch, fp.manifest_sha, fp.nthreads, fp.blas_threads, exceeded) + + println() + println("Golden update: $(case_spec.name) (v$(previous === nothing ? 0 : previous.meta.golden_version) → v$(meta.golden_version))") + if existing !== nothing + unpinnable = Set(s.name for s in case_spec.quantities if !is_pinnable(s)) + for name in sort(collect(keys(existing))) + if name in unpinnable + println(@sprintf(" %-34s REMOVED — runtimes and checksums are not pinned", name)) + elseif !haskey(values, name) + println( + @sprintf( + " %-34s REMOVED — extraction returned missing (renamed h5 path?) or the quantity left the case. This deletes its gate; confirm it is intentional.", + name + ) + ) + end + end + for (name, g) in sort(collect(values); by=first) + old = get(existing, name, nothing) + old === nothing && (println(@sprintf(" %-34s NEW", name)); continue) + old.class == g.class || + println(@sprintf(" %-34s class %s → %s (tolerances reset to provisional)", name, old.class, g.class)) + line = describe_golden_change(old, g) + line === nothing || println(@sprintf(" %-34s %s", name, line)) + end + end + if !isempty(exceeded) + names = join(sort(collect(keys(exceeded))), ", ") + accept_exceeding || error( + "Refusing to re-pin '$(case_spec.name)': $(length(exceeded)) gating quantity/quantities moved " * + "outside the old tolerance ($names). If --reason explains the move, re-run with --accept-exceeding; " * + "the old values and deviations are then recorded under [meta.exceeded]." + ) + println(" Accepted $(length(exceeded)) move(s) beyond the old tolerance, recorded under [meta.exceeded]: $names") + end + path = save_golden(meta, values) + println("Wrote $path ($(length(values)) quantities)") + provisional = count(g -> is_gating(g) && is_provisional(g), Base.values(values)) + provisional > 0 && println( + " $provisional quantity/quantities still carry provisional class-default tolerances — " * + "replace them with measured plateau drift and platform spread before relying on this as a gate." + ) + println() + return path +end + +""" +Describe how a regenerated entry's value moved from the old one, or `nothing` if it did not. + +The deviation is the one `compare_to_golden` computes against the old entry, and a move the old +tolerance would have failed is flagged so it cannot pass unnoticed as a routine re-pin. +""" +function describe_golden_change(old::GoldenValue, new::GoldenValue) + old.value_type == new.value_type || return "type $(old.value_type) → $(new.value_type)" + (old.value_real, old.value_int, old.value_text) == (new.value_real, new.value_int, new.value_text) && return nothing + q = (value_real=new.value_real, value_int=new.value_int, value_text=new.value_text, value_type=new.value_type) + passed, dev, _ = compare_to_golden(q, old) + what = if new.value_type == "real" + @sprintf("%.9g → %.9g", something(old.value_real, NaN), something(new.value_real, NaN)) + elseif new.value_type == "integer" + "$(old.value_int) → $(new.value_int)" + elseif new.value_type == "token" + "$(old.value_text) → $(new.value_text)" + else + "array[$(length(JSON.parse(new.value_text; allownan=true)))] changed" + end + rel = isnan(dev) ? "—" : @sprintf("%.2e", dev) + flag = is_gating(old) && !passed ? " ** EXCEEDS the old tolerance: a regression unless --reason explains it **" : "" + return "$what (rel $rel)$flag" +end + +""" +Build golden entries from a set of freshly extracted quantities. + +`existing` carries forward the tolerance and evidence already recorded for a quantity, so +regenerating values after a physics change does not silently reset hard-won measurements to +provisional defaults. A change of value type or of the case-declared class invalidates them. So +does a move beyond the old gating tolerance: the evidence was measured on a different value, so it +is dropped and the entry becomes provisional, with a tolerance no looser than before. +""" +function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector{QuantitySpec}, + existing::Union{Dict{String,GoldenValue},Nothing}) + spec_by_name = Dict(s.name => s for s in specs) + values = Dict{String,GoldenValue}() + for eq in extracted + eq.value_type == "missing" && continue + spec = get(spec_by_name, eq.name, nothing) + spec === nothing && continue + (is_pinnable(spec) && eq.value_type != "checksum") || continue + class = infer_class(spec) + prior = existing === nothing ? nothing : get(existing, eq.name, nothing) + if prior !== nothing && prior.value_type != eq.value_type + @warn "Golden '$(eq.name)': value_type changed $(prior.value_type) → $(eq.value_type); resetting tolerances to provisional" + prior = nothing + elseif prior !== nothing && prior.class != class + @warn "Golden '$(eq.name)': class changed $(prior.class) → $class; resetting tolerances to provisional" + prior = nothing + end + defaults = CLASS_DEFAULT_TOLERANCE[class] + run_value = (value_real=eq.value_real, value_int=eq.value_int, value_text=eq.value_text, value_type=eq.value_type) + if prior === nothing + rtol, atol, basis = defaults.rtol, defaults.atol, "class-default (provisional)" + drift, spread, at = NaN, NaN, "" + elseif is_gating(prior) && !compare_to_golden(run_value, prior)[1] + @warn "Golden '$(eq.name)': value moved beyond its old tolerance; dropping its recorded evidence and marking it provisional" + rtol, atol = min(prior.rtol, defaults.rtol), min(prior.atol, defaults.atol) + basis = (rtol, atol) == (defaults.rtol, defaults.atol) ? "class-default (provisional)" : "provisional" + drift, spread, at = NaN, NaN, "" + else + rtol, atol, basis = prior.rtol, prior.atol, prior.tolerance_basis + drift, spread, at = prior.plateau_drift, prior.platform_spread, prior.converged_at + end + # An absolute floor carried from a zero value is dropped once the value is no longer zero. + class in GATING_CLASSES && !has_exact_zero(eq.value_type, eq.value_real, eq.value_int, eq.value_text) && (atol = 0.0) + values[eq.name] = GoldenValue(eq.name, eq.value_type, eq.value_real, eq.value_int, + eq.value_text, rtol, atol, class, basis, drift, spread, at) + end + return values +end diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl index 1a192a140..ad46c0931 100644 --- a/regression-harness/src/runner.jl +++ b/regression-harness/src/runner.jl @@ -123,6 +123,62 @@ end %RUNINFO% """ + +# SLAYER dispersion Δ(Q) on a fixed 4×4 Q grid at the DIII-D-like 2/1 surface's layer parameters, +# a refinement-stable pin of the solver alone (AMR sample locations cannot be pinned; bt is unread). +const COMPUTED_SLAYER_DELTA_PROBE_SCRIPT_TEMPLATE = """ +using Pkg +%INSTANTIATE% +using GeneralizedPerturbedEquilibrium +using GeneralizedPerturbedEquilibrium.InnerLayer +using HDF5 +p = SLAYERParameters(; + tau=1.1975430647804235, lu=6.086905739791344e6, c_beta=0.22094021004591707, + D_norm=4.191469284125091, P_perp=50.2972642329308, P_tor=34.74437891503841, + Q_e=1.0915286815773122, Q_i=-1.6558720999124832, iota_e=0.39729503206497013, + tauk=0.00010358784131467763, tau_r=3.453343933553279, delta_n=504.745127277822, + rs=0.3617373814196757, R0=1.7433359412007365, bt=1.0, sval_r=1.260093929519795, + eta=4.761642777337999e-8, d_beta=0.011249087118484661) +axis = range(-10.0, 10.0; length=4) +Q = ComplexF64[re + im_ * 1im for im_ in axis for re in axis] +t_start = time() +Δ = ComplexF64[solve_inner(SLAYERModel(), p, q).tearing for q in Q] +elapsed = time() - t_start +h5open(ARGS[1], "w") do fid + fid["slayer_probe/Q_re"] = real.(Q) + fid["slayer_probe/Q_im"] = imag.(Q) + fid["slayer_probe/Delta_re"] = real.(Δ) + fid["slayer_probe/Delta_im"] = imag.(Δ) +end +%RUNINFO% +""" +# del_s over Fitzpatrick (IOP 2023) figs 6.2/6.3's (Qhat, Phat) axes: D_norm = 1, P_perp = P_tor = Phat +# and Q_e = Qhat/(1+1/tau) make the solver's Q_hat the book's. tau = 1 is assumed; Phat starts just +# inside 0, a singular edge of the model. +const COMPUTED_SLAYER_DELS_FITZPATRICK_SCRIPT_TEMPLATE = """ +using Pkg +%INSTANTIATE% +using GeneralizedPerturbedEquilibrium +using GeneralizedPerturbedEquilibrium.InnerLayer +using HDF5 +const TAU = 1.0 +probe(Q, P) = SLAYERParameters(; tau=TAU, lu=1.0, c_beta=0.0, D_norm=1.0, + P_perp=P, P_tor=P, Q_e=Q / (1 + 1/TAU), Q_i=0.0, iota_e=0.0, tauk=1.0, + tau_r=1.0, delta_n=1.0, rs=1.0, R0=1.0, bt=1.0, sval_r=1.0, eta=1.0, d_beta=1.0) +axis = [0.02, 0.5, 1.0, 2.0, 4.0] +QP = [(q, p) for p in axis for q in axis] +t_start = time() +dels = ComplexF64[riccati_del_s(probe(q, p)) for (q, p) in QP] +elapsed = time() - t_start +h5open(ARGS[1], "w") do fid + fid["fitzpatrick/Q_hat"] = Float64[q for (q, _) in QP] + fid["fitzpatrick/P_hat"] = Float64[p for (_, p) in QP] + fid["fitzpatrick/dels_db_re"] = real.(dels) + fid["fitzpatrick/dels_db_im"] = imag.(dels) + fid["fitzpatrick/tau"] = TAU +end +%RUNINFO% +""" # Self-contained separatrix-finder regression (PR #296). Loads a fixed-boundary EFIT whose # computational box hugs the LCFS (eps=0.05 TokaMaker aspect-scan g-file): outside the prescribed # LCFS the coil-vacuum flux turns back above the boundary value before the grid edge, so the old @@ -255,8 +311,12 @@ function _computed_script_template(case_spec::CaseSpec) return COMPUTED_GGJ_SCRIPT_TEMPLATE elseif case_spec.name == "ggj_ray_q500i" return COMPUTED_GGJ_RAY_SCRIPT_TEMPLATE + elseif case_spec.name == "slayer_delta_probe" + return COMPUTED_SLAYER_DELTA_PROBE_SCRIPT_TEMPLATE elseif case_spec.name == "efit_fixedbdy_separatrix" return COMPUTED_SEPARATRIX_SCRIPT_TEMPLATE + elseif case_spec.name == "slayer_dels_fitzpatrick" + return COMPUTED_SLAYER_DELS_FITZPATRICK_SCRIPT_TEMPLATE end error("No computed-script template registered for case '$(case_spec.name)'") end diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl index 05c7159ce..843d2308a 100644 --- a/regression-harness/src/types.jl +++ b/regression-harness/src/types.jl @@ -4,25 +4,30 @@ Shared data structures for the regression harness. """ Specification for a single quantity to extract from gpec.h5. + +`class` is the golden tolerance class declared in the case file; every quantity must declare one. """ struct QuantitySpec name::String h5path::String # HDF5 dataset path (e.g. "ForceFreeStates/FreeBoundaryStability/eigenmode_energies"), empty for runtime - type::String # "complex_vector", "real_vector", "real_scalar", "int_scalar", "real_matrix", "runtime" - extract::String # "value", "real_first", "imag_first", "abs_first", "norm", "all_real", "all_complex", "checksum" + type::String # "complex_vector", "real_vector", "real_scalar", "int_scalar", "real_matrix", "token", "runtime" + extract::String # "value", "real_first", "imag_first", "abs_first", "norm", "all_real", "all_complex", + # "diagonal_complex", "first_", "first__complex", "checksum", "toml_key:" label::String # Human-readable label for reports noise_threshold::Float64 # Absolute changes below this are noise order::Int # Display order in reports (lower = earlier) + class::String # Declared golden tolerance class (one of TOLERANCE_CLASSES) end """ Specification for a test case: what to run and what to extract. `kind` selects the runner backend: + - "gpec_run" (default) — run GPEC end-to-end on `example_dir`, extract from `gpec.h5` - "computed" — run a self-contained Julia computation that writes a small h5 - (no `example_dir` required); used for analytic/reference cases - like the GGJ inner-layer benchmark. + (no `example_dir` required); used for analytic/reference cases + like the GGJ inner-layer benchmark. """ struct CaseSpec name::String @@ -53,6 +58,12 @@ Parsed CLI options. worktree (pinning is on by default, so that two refs differ only by source code). `allow_env_mismatch` lets a cached result from a different environment be reused instead of re-run. `fail_on_change` turns any changed quantity into a non-zero exit status, for CI use. + +`check` runs the working tree and compares it against the committed golden values. +`update_golden` regenerates those values from a fresh run; `reason` records why, and is mandatory +because a golden change is a claim about physics that a reviewer has to be able to evaluate. +`accept_exceeding` lets such a re-pin move a gating quantity outside its old tolerance, recording +the previous value in the golden file. """ struct CLIOptions cases::Vector{String} @@ -68,5 +79,9 @@ struct CLIOptions no_pin_manifest::Bool allow_env_mismatch::Bool fail_on_change::Bool + check::Bool + update_golden::Bool + accept_exceeding::Bool + reason::Union{String,Nothing} help::Bool end diff --git a/regression-harness/test/runtests.jl b/regression-harness/test/runtests.jl new file mode 100644 index 000000000..6906a8705 --- /dev/null +++ b/regression-harness/test/runtests.jl @@ -0,0 +1,573 @@ +# Unit tests for the golden-value comparison logic (src/golden.jl), on synthetic data only — +# no GPEC runs. Run against the harness project, not the GPEC package: +# +# julia --project=regression-harness regression-harness/test/runtests.jl +# +using Test +using Dates, HDF5, JSON, Printf, SHA, SQLite, Tables, TOML + +const HARNESS_DIR = abspath(joinpath(@__DIR__, "..")) +const REPO_ROOT = abspath(joinpath(HARNESS_DIR, "..")) +const DEFAULT_DB_PATH = joinpath(mktempdir(), "test_cache.sqlite") +const CASES_DIR = joinpath(HARNESS_DIR, "cases") +# Redirected to a temp dir so tests can write and corrupt golden files freely without ever +# touching the tracked pins under regression-harness/golden/. +const GOLDEN_DIR = mktempdir() + +for f in ("types.jl", "env.jl", "config.jl", "database.jl", "utils.jl", "extractor.jl", "runner.jl", "reporter.jl", "golden.jl") + include(joinpath(HARNESS_DIR, "src", f)) +end + +qspec(name; type="real_scalar", extract="value", label=name, noise=0.0, order=1, class="physics_converged") = + QuantitySpec(name, "", type, extract, label, noise, order, class) + +golden_val(name; value_type="real", value_real=nothing, value_int=nothing, value_text=nothing, + rtol=1e-6, atol=0.0, class="physics_converged", basis="class-default (provisional)", drift=NaN, spread=NaN, at="") = + GoldenValue(name, value_type, value_real, value_int, value_text, rtol, atol, class, basis, drift, spread, at) + +qtuple(; value_real=nothing, value_int=nothing, value_text=nothing, value_type="real", label="q") = + (label=label, value_real=value_real, value_int=value_int, value_text=value_text, value_type=value_type, noise_threshold=0.0) + +extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, value_type="real") = + ExtractedQuantity(name, name, value_real, value_int, value_text, value_type, 0.0) + +@testset "golden comparison logic" begin + + @testset "scalar real comparison" begin + g = golden_val("x"; value_real=1.0, rtol=1e-6) + passed, dev, detail = compare_to_golden(qtuple(; value_real=1.0 + 5e-7), g) + @test passed + @test dev ≈ 5e-7 rtol = 1e-3 + @test isempty(detail) + + passed, dev, detail = compare_to_golden(qtuple(; value_real=1.0 + 1e-5), g) + @test !passed + @test dev ≈ 1e-5 rtol = 1e-3 + @test !isempty(detail) + + # Zero golden value: rtol contributes nothing, deviation is reported absolute. + g0 = golden_val("x"; value_real=0.0, rtol=1e-6, atol=0.0) + passed, dev, _ = compare_to_golden(qtuple(; value_real=1e-12), g0) + @test !passed + @test dev ≈ 1e-12 + g0a = golden_val("x"; value_real=0.0, rtol=1e-6, atol=1e-9) + @test compare_to_golden(qtuple(; value_real=1e-12), g0a)[1] + + # Non-gating classes are never judged, even on a zero golden value where Inf*0 = NaN. + gd = golden_val("x"; value_real=0.0, rtol=Inf, atol=Inf, class="diagnostic") + @test compare_to_golden(qtuple(; value_real=42.0), gd)[1] + # Finite tolerances on a non-gating class do not make it gate either. + @test compare_to_golden(qtuple(; value_real=42.0), golden_val("x"; value_real=1.0, class="unconverged"))[1] + + @test !compare_to_golden(qtuple(; value_real=nothing), g)[1] + # A NaN run value fails a finite gold rather than slipping through a false comparison. + passed, dev, _ = compare_to_golden(qtuple(; value_real=NaN), g) + @test !passed && isnan(dev) + end + + @testset "integer and token comparison" begin + # Topological integers gate on exact equality no matter how loose the rtol. + gt = golden_val("n"; value_type="integer", value_int=5, rtol=10.0, class="topological") + @test compare_to_golden(qtuple(; value_int=5, value_type="integer"), gt)[1] + @test !compare_to_golden(qtuple(; value_int=6, value_type="integer"), gt)[1] + + # Non-topological integers use the tolerance like a real. + gi = golden_val("n"; value_type="integer", value_int=100, rtol=0.05, class="physics_converged") + @test compare_to_golden(qtuple(; value_int=103, value_type="integer"), gi)[1] + @test !compare_to_golden(qtuple(; value_int=110, value_type="integer"), gi)[1] + + # Tokens are discrete: deviation 0.0 on match, 1.0 (not NaN) on mismatch so it sorts as + # a real failure. + gk = golden_val("integ"; value_type="token", value_text="riccati", class="topological") + passed, dev, _ = compare_to_golden(qtuple(; value_text="riccati", value_type="token"), gk) + @test passed && dev == 0.0 + passed, dev, detail = compare_to_golden(qtuple(; value_text="galerkin", value_type="token"), gk) + @test !passed && dev == 1.0 + @test occursin("riccati", detail) && occursin("galerkin", detail) + end + + @testset "array worst-element path" begin + g = golden_val("v"; value_type="json_array", value_text="[1.0,2.0,4.0]", rtol=1e-6) + + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0000000001,2.0,4.0]", value_type="json_array"), g) + @test passed + @test dev ≈ 1e-10 rtol = 1e-3 + @test isempty(detail) + + # One element out of tolerance fails the whole array and the detail names it. + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0,2.002,4.0]", value_type="json_array"), g) + @test !passed + @test dev ≈ 1e-3 rtol = 1e-3 + @test occursin("worst element 2", detail) + + # The reported deviation is the worst RELATIVE element, not the first failing one. + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.001,2.0,4.02]", value_type="json_array"), g) + @test !passed + @test dev ≈ 5e-3 rtol = 1e-3 + @test occursin("worst element 3", detail) + + # A zero golden element is judged absolutely; atol=0 makes any movement on it fail. + gz = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=1e-6, atol=0.0) + @test !compare_to_golden(qtuple(; value_text="[1.0e-9,1.0]", value_type="json_array"), gz)[1] + gza = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=1e-6, atol=1e-8) + @test compare_to_golden(qtuple(; value_text="[1.0e-9,1.0]", value_type="json_array"), gza)[1] + + # Non-finite tolerances never judge, including zero golden elements (the Inf*0 = NaN guard). + gdz = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=Inf, atol=Inf, class="diagnostic") + @test compare_to_golden(qtuple(; value_text="[5.0,99.0]", value_type="json_array"), gdz)[1] + + # Complex encoding: [re, im] pairs go through the same worst-element machinery. + gc = golden_val("v"; value_type="json_array", value_text="[[1.0,0.5],[2.0,-1.0]]", rtol=1e-6) + @test compare_to_golden(qtuple(; value_text="[[1.0,0.5],[2.0,-1.0]]", value_type="json_array"), gc)[1] + passed, _, detail = compare_to_golden(qtuple(; value_text="[[1.0,0.5],[2.0,-1.01]]", value_type="json_array"), gc) + @test !passed + @test occursin("worst element 2", detail) + + passed, _, detail = compare_to_golden(qtuple(; value_text="[1.0,2.0]", value_type="json_array"), g) + @test !passed + @test occursin("length", detail) + + # A NaN element fails and is the element the detail names; NaN on both sides is "no result" + # in both runs and matches. + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0,NaN,4.0]", value_type="json_array"), g) + @test !passed && isinf(dev) + @test occursin("worst element 2", detail) + gn = golden_val("v"; value_type="json_array", value_text="[1.0,NaN]", rtol=1e-6) + @test compare_to_golden(qtuple(; value_text="[1.0,NaN]", value_type="json_array"), gn)[1] + + # A nested row that changed length fails instead of being truncated by zip. + gr = golden_val("m"; value_type="json_array", value_text="[[1.0,2.0],[3.0,4.0]]", rtol=1e-6) + @test !compare_to_golden(qtuple(; value_text="[[1.0,2.0],[3.0]]", value_type="json_array"), gr)[1] + end + + @testset "type-change handling" begin + g = golden_val("x"; value_real=1.0) + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0]", value_type="json_array"), g) + @test !passed && isnan(dev) + @test occursin("type changed", detail) + + # A value_type change through build_golden_values resets class and tolerances to + # provisional: measurements made under the old type must not carry over. + spec = qspec("x") + prior = Dict("x" => golden_val("x"; value_type="integer", value_int=5, rtol=0.0, + class="topological", basis="measured", drift=1e-8, spread=1e-7, at="mpsi=1024")) + vals = @test_logs (:warn, r"value_type changed") build_golden_values( + [extracted_q("x"; value_real=5.0)], [spec], prior) + @test vals["x"].class == "physics_converged" + @test vals["x"].rtol == CLASS_DEFAULT_TOLERANCE["physics_converged"].rtol + @test vals["x"].tolerance_basis == "class-default (provisional)" + @test isnan(vals["x"].plateau_drift) && isnan(vals["x"].platform_spread) + + # Same type, moved within the old tolerance: the prior's measurements carry forward unchanged. + prior2 = Dict("x" => golden_val("x"; value_real=1.0, rtol=3e-7, atol=1e-12, + basis="measured", drift=1e-8, spread=2e-7, at="mpsi=1024")) + vals2 = build_golden_values([extracted_q("x"; value_real=1.0 + 1e-7)], [spec], prior2) + @test vals2["x"].value_real == 1.0 + 1e-7 + # An absolute floor on a nonzero value is not a legal gate, so it is dropped rather than carried. + @test vals2["x"].rtol == 3e-7 && vals2["x"].atol == 0.0 + @test vals2["x"].tolerance_basis == "measured" + @test vals2["x"].plateau_drift == 1e-8 && vals2["x"].platform_spread == 2e-7 + + # A move beyond the old tolerance drops the evidence, and the tolerance never loosens. + vals5 = @test_logs (:warn, r"beyond its old tolerance") build_golden_values( + [extracted_q("x"; value_real=1.5)], [spec], prior2) + @test vals5["x"].value_real == 1.5 + @test vals5["x"].rtol == 3e-7 && vals5["x"].atol == 0.0 + @test vals5["x"].tolerance_basis == "provisional" && is_provisional(vals5["x"]) + @test isnan(vals5["x"].plateau_drift) && isnan(vals5["x"].platform_spread) && isempty(vals5["x"].converged_at) + + # A structural zero keeps its absolute floor across a re-pin that stays zero. + priorz = Dict("z" => golden_val("z"; value_type="json_array", value_text="[0.0,1.0]", atol=1e-10)) + valsz = build_golden_values([extracted_q("z"; value_text="[0.0,1.0000001]", value_type="json_array")], [qspec("z"; type="real_array")], priorz) + @test valsz["z"].atol == 1e-10 + # A measured tolerance wider than the class default is tightened back to the default. + prior5 = Dict("x" => golden_val("x"; value_real=1.0, rtol=1e-4, basis="measured", drift=5e-5)) + vals6 = @test_logs (:warn, r"beyond its old tolerance") build_golden_values( + [extracted_q("x"; value_real=1.5)], [spec], prior5) + @test vals6["x"].rtol == CLASS_DEFAULT_TOLERANCE["physics_converged"].rtol + @test vals6["x"].tolerance_basis == "class-default (provisional)" + + # A change of the case-declared class resets measurements, like a type change. + prior3 = Dict("x" => golden_val("x"; value_real=1.0, rtol=3e-7, basis="measured", drift=1e-8)) + vals4 = @test_logs (:warn, r"class changed") build_golden_values( + [extracted_q("x"; value_real=1.0)], [qspec("x"; class="equilibrium_scalar")], prior3) + @test vals4["x"].class == "equilibrium_scalar" + @test vals4["x"].tolerance_basis == "class-default (provisional)" + + # Missing extractions, checksums, runtimes, and quantities without a spec never become entries. + vals3 = build_golden_values( + [extracted_q("gone"; value_type="missing"), + extracted_q("x"; value_text="abc123", value_type="checksum"), + extracted_q("rt"; value_real=12.5), + extracted_q("unspecced"; value_real=1.0)], + [spec, qspec("rt"; type="runtime", class="diagnostic")], nothing) + @test isempty(vals3) + end + + @testset "load-validation refusals" begin + write( + golden_path("bad_class"), + """ +[values.x] +class = "definitely_not_a_class" +rtol = 1.0e-6 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("bad_class") + + # A gating entry with no finite rtol is malformed whether it was hand-edited or merged. + write( + golden_path("no_rtol"), + """ +[values.x] +class = "physics_converged" +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("no_rtol") + + # An rtol below the recorded platform spread is a gate no second platform can pass; + # enforced on every load so a hand edit cannot ship what save_golden refused to write. + write( + golden_path("too_tight"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-9 +platform_spread = 1.0e-6 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("too_tight") + + # A non-finite atol on a gating class would make every comparison pass. + for bad in ("inf", "nan") + write( + golden_path("atol_$bad"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-6 +atol = $bad +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("atol_$bad") + end + + # An rtol below the measured plateau drift is a gate the converged answer itself fails. + write( + golden_path("below_drift"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-9 +plateau_drift = 1.0e-7 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("below_drift") + + # atol on a gating entry is only for structural zeros. + write( + golden_path("atol_nonzero"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-6 +atol = 1.0e-12 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("atol_nonzero") + write( + golden_path("atol_zeros"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-6 +atol = 1.0e-12 +value = 0.0 + +[values.v] +class = "physics_converged" +value_type = "json_array" +value_text = "[1.0,0.0]" +rtol = 1.0e-6 +atol = 1.0e-12 + +[values.c] +class = "physics_converged" +value_type = "json_array" +value_text = "[[1.0,0.0],[0.0,0.0]]" +rtol = 1.0e-6 +atol = 1.0e-12 +""" + ) + @test length(load_golden("atol_zeros").values) == 3 + # A complex pair with only one zero component is not a structural zero. + @test !has_exact_zero(golden_val("c"; value_type="json_array", value_text="[[1.0,0.0],[2.0,0.5]]")) + @test has_exact_zero(golden_val("n"; value_type="integer", value_int=0)) + @test_throws ErrorException save_golden(GoldenMeta("atol_save", 1, "", "", "", "", "", "", 1, 1), + Dict("x" => golden_val("x"; value_real=2.0, atol=1e-9))) + + # "measured" must be backed by a recorded measurement. + write( + golden_path("unbacked"), + """ +[values.x] +class = "physics_converged" +tolerance_basis = "measured" +rtol = 1.0e-6 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("unbacked") + + # Non-gating classes are exempt: they are recorded, never judged. + write( + golden_path("diag_inf"), + """ +[values.rt] +class = "diagnostic" +rtol = inf +atol = inf +value = 12.5 +""" + ) + loaded = load_golden("diag_inf") + @test loaded !== nothing && !is_gating(loaded.values["rt"]) + + @test load_golden("no_such_case") === nothing + @test !has_golden("no_such_case") + end + + @testset "save_golden refusals and round trip" begin + meta = GoldenMeta("rt_case", 1, "2026-08-31", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4) + + @test_throws ErrorException save_golden(meta, Dict("empty" => golden_val("empty"))) + @test_throws ErrorException save_golden(meta, + Dict("tight" => golden_val("tight"; value_real=1.0, rtol=1e-9, spread=1e-6))) + @test_throws ErrorException save_golden(meta, + Dict("open" => golden_val("open"; value_real=1.0, rtol=1e-6, atol=Inf, spread=1e-7))) + + vals = Dict( + "a" => golden_val("a"; value_real=2.5, rtol=1e-6, drift=1e-8, spread=1e-7, at="mpsi=512"), + "n" => golden_val("n"; value_type="integer", value_int=7, rtol=0.0, class="topological"), + "v" => golden_val("v"; value_type="json_array", value_text="[1.0,2.0]", rtol=1e-6)) + save_golden(meta, vals) + back = load_golden("rt_case") + @test back.meta.commit == "deadbeef" && back.meta.golden_version == 1 + @test back.values["a"].value_real == 2.5 + @test back.values["a"].rtol == 1e-6 + @test back.values["a"].plateau_drift == 1e-8 && back.values["a"].platform_spread == 1e-7 + @test back.values["n"].value_int == 7 && back.values["n"].class == "topological" + @test back.values["v"].value_text == "[1.0,2.0]" + @test isempty(back.meta.exceeded) + + # Accepted moves beyond the old tolerance round-trip through [meta.exceeded]. + exc = exceeding_changes(Dict("a" => golden_val("a"; value_real=2.5)), Dict("a" => golden_val("a"; value_real=2.6)), 1) + meta2 = GoldenMeta("rt_case", 2, "2026-09-25", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4, exc) + save_golden(meta2, Dict("a" => golden_val("a"; value_real=2.6))) + back2 = load_golden("rt_case") + @test back2.meta.exceeded["a"]["previous"] == 2.5 && back2.meta.exceeded["a"]["previous_version"] == 1 + @test back2.meta.exceeded["a"]["deviation"] ≈ 0.04 + @test occursin("[meta.exceeded.a]", read(golden_path("rt_case"), String)) + end + + @testset "exceeding_changes" begin + existing = Dict( + "in" => golden_val("in"; value_real=1.0), + "out" => golden_val("out"; value_real=1.0), + "diag" => golden_val("diag"; value_real=1.0, rtol=Inf, atol=Inf, class="diagnostic"), + "typ" => golden_val("typ"; value_real=1.0), + "tok" => golden_val("tok"; value_type="token", value_text="riccati", class="topological")) + values = Dict( + "in" => golden_val("in"; value_real=1.0 + 1e-7), + "out" => golden_val("out"; value_real=1.1), + "diag" => golden_val("diag"; value_real=50.0, rtol=Inf, atol=Inf, class="diagnostic"), + "typ" => golden_val("typ"; value_type="json_array", value_text="[1.0]"), + "tok" => golden_val("tok"; value_type="token", value_text="galerkin", class="topological"), + "new" => golden_val("new"; value_real=3.0)) + exc = exceeding_changes(existing, values, 4) + @test sort(collect(keys(exc))) == ["out", "tok", "typ"] + @test exc["out"]["previous"] == 1.0 && exc["out"]["deviation"] ≈ 0.1 && exc["out"]["previous_version"] == 4 + @test exc["tok"]["previous"] == "riccati" + @test isnan(exc["typ"]["deviation"]) + end + + @testset "update refuses a move beyond the old tolerance" begin + repo = mktempdir() + run(`git -C $repo init -q`) + write(joinpath(repo, "f.txt"), "x\n") + run(`git -C $repo add -A`) + run(`git -C $repo -c user.name=t -c user.email=t@t commit -q -m init`) + sha = String(strip(read(`git -C $repo rev-parse HEAD`, String))) + db = open_database(joinpath(mktempdir(), "update.sqlite")) + case = CaseSpec("exceed_case", "synthetic", "", [qspec("a"), qspec("b")], "example", Dict{String,Any}()) + save_golden(GoldenMeta("exceed_case", 3, "2026-09-01", "cafe", "seed", "1.11.6", "arm64", "abc", 4, 4), + Dict("a" => golden_val("a"; value_real=1.0), "b" => golden_val("b"; value_real=2.0))) + store_run(db, sha, sha, "", "", "exceed_case", 1.0, [extracted_q("a"; value_real=1.2), extracted_q("b"; value_real=2.0)]) + @test_throws ErrorException redirect_stdout(devnull) do + update_golden_from_run(db, case, sha, "moved", repo) + end + @test load_golden("exceed_case").meta.golden_version == 3 + @test_logs (:warn, r"beyond its old tolerance") match_mode = :any redirect_stdout(devnull) do + update_golden_from_run(db, case, sha, "moved", repo; accept_exceeding=true) + end + back = load_golden("exceed_case") + @test back.meta.golden_version == 4 && back.values["a"].value_real == 1.2 + @test collect(keys(back.meta.exceeded)) == ["a"] + @test back.meta.exceeded["a"]["previous"] == 1.0 && back.meta.exceeded["a"]["previous_version"] == 3 + close_database(db) + end + + @testset "infer_class" begin + # The declared class is returned as-is, whatever the quantity's name or type suggests. + for class in TOLERANCE_CLASSES + @test infer_class(qspec("x"; class=class)) == class + end + @test infer_class(qspec("q0"; class="physics_converged")) == "physics_converged" + @test infer_class(qspec("nstep"; type="int_scalar", class="topological")) == "topological" + # A missing or unknown class is an error, never a default. + @test_throws ErrorException infer_class(qspec("x"; class="")) + @test_throws ErrorException infer_class(qspec("x"; class="gating_optional")) + end + + @testset "load_case requires a declared class" begin + dir = mktempdir() + write_case(body) = (path = joinpath(dir, "c.toml"); write(path, "[case]\nname = \"c\"\n\n" * body); path) + block = "[quantities.a]\nh5path = \"x\"\ntype = \"real_scalar\"\nextract = \"value\"\n" + @test load_case(write_case(block * "class = \"diagnostic\"\n")).quantities[1].class == "diagnostic" + err = try + load_case(write_case(block)) + "" + catch e + sprint(showerror, e) + end + @test occursin("c.toml", err) && occursin("'a'", err) && occursin("no `class`", err) + @test_throws ErrorException load_case(write_case(block * "class = \"gating_optional\"\n")) + # Every committed case declares a valid class on every quantity. + @test all(q -> q.class in TOLERANCE_CLASSES, Iterators.flatten(c.quantities for c in values(load_all_cases(CASES_DIR)))) + end + + @testset "is_pinnable" begin + @test is_pinnable(qspec("a")) + @test !is_pinnable(qspec("rt"; type="runtime", class="diagnostic")) + @test !is_pinnable(qspec("h"; extract="checksum")) + end + + @testset "golden check counting and crash classification" begin + db = open_database(joinpath(mktempdir(), "check.sqlite")) + case = CaseSpec("count_case", "synthetic", "", + [ + qspec("a"), + qspec("rt"; type="runtime", extract="", class="diagnostic"), + qspec("c"; type="int_scalar", class="topological"), + qspec("d"; class="diagnostic"), + qspec("e"), + qspec("f"), + qspec("demoted")], "example", Dict{String,Any}()) + + meta = GoldenMeta("count_case", 1, "2026-08-31", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4) + save_golden( + meta, + Dict( + "a" => golden_val("a"; value_real=1.0, rtol=1e-6), + "c" => golden_val("c"; value_type="integer", value_int=5, rtol=0.0, class="topological"), + "d" => golden_val("d"; value_real=3.0, rtol=Inf, atol=Inf, class="diagnostic"), + "f" => golden_val("f"; value_real=9.0, rtol=1e-6), + "demoted" => golden_val("demoted"; value_real=2.0, rtol=Inf, atol=Inf, class="unconverged"), + "orphan" => golden_val("orphan"; value_real=7.0, rtol=1e-6)) + ) + + # a passes, c fails (5 → 6), d moves wildly but is declared diagnostic, e has no golden, + # f is golden-pinned but missing from the run, demoted is non-gating in the golden file only, + # orphan has no spec left in the case. + store_run( + db, + "hash1", + "hash1", + "", + "", + "count_case", + 1.0, + [ + extracted_q("a"; value_real=1.0 + 1e-8), + extracted_q("rt"; value_real=33.0), + extracted_q("c"; value_int=6, value_type="integer"), + extracted_q("d"; value_real=300.0), + extracted_q("e"; value_real=1.0), + extracted_q("demoted"; value_real=50.0)] + ) + s = report_golden_check(db, case, "hash1") + @test s.n_pass == 1 + @test s.n_fail == 4 # c mismatch + f missing + class demoted outside the case + orphan + @test s.n_informational == 1 + @test s.n_untracked == 1 # e only: an unpinned runtime is structurally un-goldenable + @test s.n_run_failed == 0 + + # A crashed run is a crash, not a tolerance failure: n_run_failed, never n_fail. + store_failed_run(db, "hash2", "hash2", "", "", "count_case", "boom: solver exploded") + s = report_golden_check(db, case, "hash2") + @test s.n_run_failed == 1 + @test s.n_fail == 0 && s.n_pass == 0 + + # No golden file: every count zero, so the caller's zero-coverage guard (a --check that + # gated nothing must exit red) can see that nothing was checked. + nocase = CaseSpec("never_pinned", "synthetic", "", [qspec("a")], "example", Dict{String,Any}()) + s = report_golden_check(db, nocase, "hash1") + @test s == (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=0) + + # A golden file holding only non-gating entries gates nothing, and the counts say so. + diagcase = CaseSpec("diag_only", "synthetic", "", [qspec("d"; class="diagnostic")], "example", Dict{String,Any}()) + save_golden(GoldenMeta("diag_only", 1, "2026-08-31", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4), + Dict("d" => golden_val("d"; value_real=3.0, rtol=Inf, atol=Inf, class="diagnostic"))) + store_run(db, "hash3", "hash3", "", "", "diag_only", 1.0, [extracted_q("d"; value_real=30.0)]) + s = report_golden_check(db, diagcase, "hash3") + @test s.n_pass + s.n_fail == 0 && s.n_informational == 1 + close_database(db) + end + + @testset "re-pin reporting" begin + old = golden_val("a"; value_real=1.0, rtol=1e-6) + @test describe_golden_change(old, golden_val("a"; value_real=1.0, rtol=1e-6)) === nothing + @test !occursin("EXCEEDS", describe_golden_change(old, golden_val("a"; value_real=1.0 + 1e-7))) + @test occursin("EXCEEDS", describe_golden_change(old, golden_val("a"; value_real=1.1))) + # Integer and token moves print their values, not just "changed". + oi = golden_val("n"; value_type="integer", value_int=5, rtol=0.0, class="topological") + line = describe_golden_change(oi, golden_val("n"; value_type="integer", value_int=4, rtol=0.0, class="topological")) + @test occursin("5 → 4", line) && occursin("EXCEEDS", line) + ok = golden_val("k"; value_type="token", value_text="riccati", class="topological") + @test occursin("riccati → galerkin", describe_golden_change(ok, golden_val("k"; value_type="token", value_text="galerkin", class="topological"))) + # Moves on a non-gating entry are shown but never flagged. + od = golden_val("d"; value_real=1.0, rtol=Inf, atol=Inf, class="diagnostic") + @test !occursin("EXCEEDS", describe_golden_change(od, golden_val("d"; value_real=9.0, rtol=Inf, atol=Inf, class="diagnostic"))) + end + + @testset "update refuses uncommitted source" begin + repo = mktempdir() + run(`git -C $repo init -q`) + mkpath(joinpath(repo, "golden")) + write(joinpath(repo, "src.jl"), "x = 1\n") + write(joinpath(repo, "golden", "case.toml"), "a = 1\n") + run(`git -C $repo add -A`) + run(`git -C $repo -c user.name=t -c user.email=t@t commit -q -m init`) + @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) + # Rewriting a golden file is the update itself, and untracked files outside the source dirs are not source. + write(joinpath(repo, "golden", "case.toml"), "a = 2\n") + write(joinpath(repo, "scratch.md"), "notes\n") + @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) + # An untracked file under a source dir can change the run, so it is refused like a tracked edit. + for f in ("src/new.jl", "examples/case/gpec.toml", "regression-harness/cases/new_case.toml") + mkpath(dirname(joinpath(repo, f))) + write(joinpath(repo, f), "x\n") + @test occursin(f, uncommitted_changes(repo, joinpath(repo, "golden"))) + rm(joinpath(repo, f)) + end + @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) + write(joinpath(repo, "src.jl"), "x = 2\n") + @test occursin("src.jl", uncommitted_changes(repo, joinpath(repo, "golden"))) + end +end