From c3e53147f59add15c604dc89415785bf8f207372 Mon Sep 17 00:00:00 2001 From: d-burg Date: Sat, 15 Aug 2026 02:22:57 -0400 Subject: [PATCH 01/30] docs: chi_tor is Fitzpatrick's chi_phi (momentum viscosity), not heat Three docstrings called chi_tor a 'toroidal heat diffusivity'. It is Fitzpatrick's chi_phi, the anomalous perpendicular ion MOMENTUM diffusivity: in the TJ four-field layer model (TJ/Documentation/Layer.tex) it enters the vorticity equation as chi_phi*grad^4(phi + iota_i*N) and the parallel-flow equation as chi_phi*grad^2(V) -- a viscosity in both places -- with tau_phi = r_s^2/chi_phi named the toroidal momentum confinement time and P_phi = tau_R/tau_phi a magnetic Prandtl number. The codebase already knew: the DIII-D example toml comments 'the kinetic file supplies chi_phi(psi)' for this knob, Control.jl maps it to the chi_phi profile dataset, and P_tor/P_perp printed by a run equals chi_tor/chi_perp exactly. Only the fallback-scalar docstrings were wrong. Found while trying to match M3D-C1's viscosity (amu) to SLAYER for the TJ benchmark: the mislabel made SLAYER appear to carry no momentum-viscosity input at all, which cost a day of analysis concluding the wrong thing. Co-Authored-By: Claude Opus 5 --- src/InnerLayer/SLAYER/LayerInputs.jl | 6 ++++-- src/InnerLayer/SLAYER/LayerParameters.jl | 7 +++++-- src/Tearing/Runner/Control.jl | 11 +++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/InnerLayer/SLAYER/LayerInputs.jl b/src/InnerLayer/SLAYER/LayerInputs.jl index 252e81ae9..92664c207 100644 --- a/src/InnerLayer/SLAYER/LayerInputs.jl +++ b/src/InnerLayer/SLAYER/LayerInputs.jl @@ -183,8 +183,10 @@ profiles, without an intermediate file round-trip. - `zeff` -- effective charge (default `1.0`). - `chi_perp` -- perpendicular heat diffusivity [m²/s]. Scalar or a callable of `psi` (default `1.0`). - - `chi_tor` -- toroidal heat diffusivity [m²/s]. Scalar or a callable - of `psi` (default `1.0`). + - `chi_tor` -- Fitzpatrick's χ_φ: anomalous perpendicular ion MOMENTUM + diffusivity [m²/s] (a viscosity entering the vorticity and parallel-flow + equations — not a heat diffusivity). Scalar or a callable of `psi` + (default `1.0`). - `dr_val` -- resistive interchange index `D_R = E + F + H²` (Glasser-Greene-Johnson 1975) feeding the critical-Δ formulas (`:lar`, `:rfitzp`, `:toroidal`). When `nothing` (default), Julia diff --git a/src/InnerLayer/SLAYER/LayerParameters.jl b/src/InnerLayer/SLAYER/LayerParameters.jl index ae875dea4..a0ba39414 100644 --- a/src/InnerLayer/SLAYER/LayerParameters.jl +++ b/src/InnerLayer/SLAYER/LayerParameters.jl @@ -29,7 +29,7 @@ de-normalization. The parametrization uses `P_perp`, `P_tor`, and | `c_beta` | Compressibility √(β_local / (1 + β_local)) | | `D_norm` | (d_β/r_s) · S^(1/3) · √ι_e (Fitzpatrick normalized scale) | | `P_perp` | Perpendicular Prandtl number τ_R / τ_⊥ | -| `P_tor` | Toroidal-direction Prandtl number τ_R / τ_‖tor | +| `P_tor` | Momentum (magnetic) Prandtl number τ_R / τ_φ, τ_φ = r_s²/χ_φ | | `Q_e` | Normalized electron diamagnetic: −tauk · ω_*e | | `Q_i` | Normalized ion diamagnetic: −tauk · ω_*i | | `iota_e` | Q_e / (Q_e − Q_i) | @@ -210,7 +210,10 @@ parametrization (P_perp/P_tor/D_norm; the older magnetic/electron Prandtl - `R0` -- major radius [m] - `mu_i` -- ion mass in proton-mass units (e.g. 2.0 for D) - `zeff` -- effective charge - - `chi_perp`, `chi_tor` -- perpendicular / toroidal heat diffusivity [m²/s] + - `chi_perp` -- perpendicular energy diffusivity χ⊥ [m²/s] + - `chi_tor` -- Fitzpatrick's χ_φ, the anomalous perpendicular ion momentum + diffusivity [m²/s] (a viscosity, not heat; TJ Layer.tex eqs. — enters the + vorticity and parallel-flow equations, τ_φ = r_s²/χ_φ) - `m`, `n` -- poloidal / toroidal mode numbers at the surface - `dr_val`, `dgeo_val` -- inputs for the critical-Δ formula - `dc_type` -- one of `:none`, `:lar`, `:rfitzp`, `:toroidal` diff --git a/src/Tearing/Runner/Control.jl b/src/Tearing/Runner/Control.jl index 1a12ff7e6..f41777468 100644 --- a/src/Tearing/Runner/Control.jl +++ b/src/Tearing/Runner/Control.jl @@ -34,10 +34,13 @@ constructor. callable of `psi` overrides it - `mu_i` -- ion mass in proton-mass units (default 2.0 for D) - `zeff` -- effective charge - - `chi_perp`, `chi_tor` -- fallback perpendicular / toroidal heat - diffusivity [m²/s], used only when the kinetic file carries no usable - `chi_e`/`chi_phi` profile (dataset absent or all-zero); otherwise the - file's χ⊥(ψ)/χ_φ(ψ) take precedence + - `chi_perp`, `chi_tor` -- fallback scalars [m²/s], used only when the + kinetic file carries no usable `chi_e`/`chi_phi` profile (dataset absent + or all-zero); otherwise the file's χ⊥(ψ)/χ_φ(ψ) take precedence. + `chi_perp` is the perpendicular ENERGY diffusivity χ⊥; `chi_tor` is + Fitzpatrick's χ_φ, the anomalous perpendicular ion MOMENTUM diffusivity + (a viscosity: it enters the vorticity and parallel-flow equations, TJ + Layer.tex — not a heat diffusivity) - `dr_val`, `dgeo_val` -- critical-Δ formula inputs. `nothing` (default) auto-derives them from the equilibrium: `dr_val` from the resistive interchange index `D_R = E + F + H²` at each surface, `dgeo_val` from the From f027b4f1d07357d26fa7b0e7eea0c2ecd76f6812 Mon Sep 17 00:00:00 2001 From: d-burg Date: Sat, 15 Aug 2026 17:12:43 -0400 Subject: [PATCH 02/30] REGRESSION - NEW FEATURE - Golden values: committed pins with derived tolerances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Differential --refs comparison answers "did this change?" but not "is this right?": nothing is recorded in git and both sides come from the same code. Golden values close that gap — one tracked TOML per case under regression-harness/golden/, holding each quantity's value, the tolerance it must reproduce within (pass iff |x - gold| <= atol + rtol*|gold|), and the evidence behind that tolerance. Five tolerance classes set policy: topological (exact), equilibrium_scalar (tight), physics_converged (measured plateau drift + platform spread), diagnostic and unconverged (reported, never gating — a quantity with no plateau is visibly excluded rather than quietly given a wide bound). Checksums are excluded: a bitwise hash has no notion of close, so it cannot carry a tolerance. Tolerances are derived, never chosen: save_golden refuses an rtol tighter than the recorded platform spread, every entry carries tolerance_basis so a provisional class-default can never be mistaken for a measured bound, and regeneration carries prior evidence forward instead of resetting it. CLI: --check gates the working tree against the committed values (exit 1 on any gating failure); --update-golden regenerates them and requires --reason, because a golden change is a claim about physics a reviewer has to evaluate. Reports print the golden's provenance (commit, environment, reason) next to the current run's. Co-Authored-By: Claude Fable 5 --- regression-harness/regress.jl | 83 +++++- regression-harness/src/golden.jl | 456 +++++++++++++++++++++++++++++++ regression-harness/src/types.jl | 8 + 3 files changed, 538 insertions(+), 9 deletions(-) create mode 100644 regression-harness/src/golden.jl diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 651a3df8d..f279bd6b6 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -6,6 +6,7 @@ const HARNESS_DIR = @__DIR__ const REPO_ROOT = abspath(joinpath(HARNESS_DIR, "..")) const DEFAULT_DB_PATH = joinpath(HARNESS_DIR, ".regress_cache.sqlite") const CASES_DIR = joinpath(HARNESS_DIR, "cases") +const GOLDEN_DIR = joinpath(HARNESS_DIR, "golden") include("src/types.jl") include("src/env.jl") @@ -15,6 +16,7 @@ include("src/utils.jl") include("src/extractor.jl") include("src/runner.jl") include("src/reporter.jl") +include("src/golden.jl") function parse_args(args) cases = String[] @@ -30,6 +32,9 @@ function parse_args(args) no_pin_manifest = false allow_env_mismatch = false fail_on_change = false + check = false + update_golden = false + reason = nothing help = false i = 1 @@ -59,6 +64,15 @@ function parse_args(args) elseif arg == "--fail-on-change" fail_on_change = true i += 1 + elseif arg == "--check" + check = true + i += 1 + elseif arg == "--update-golden" + update_golden = true + i += 1 + elseif arg == "--reason" && i < length(args) + reason = args[i+1] + i += 2 elseif arg == "--cases" && i < length(args) cases = split(args[i+1], ",") |> collect .|> strip i += 2 @@ -83,7 +97,7 @@ function parse_args(args) end return CLIOptions(cases, refs, ref_range, force, list_cases, show_qty, show_case, db_path, verbose, - no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, help) + no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, check, update_golden, reason, help) end const HELP_TEXT = """ @@ -117,15 +131,26 @@ Options: re-running them --fail-on-change Exit non-zero if any tracked quantity changed (for CI use; a failed run always exits non-zero regardless) + --check Run the working tree and compare against the committed golden values + (regression-harness/golden/.toml). This is the mode CI gates on. + --update-golden Regenerate the golden values from a fresh run. Requires --reason. + --reason "..." Why the goldens changed. Mandatory with --update-golden: a golden + change is a claim about physics that a reviewer has to evaluate. --help Print this help message Environment: GPEC_REGRESS_THREADS Threads for GPEC subprocesses (default "auto" = all cores); the actual count is recorded in each run's env fingerprint +Golden values: + A golden file pins each quantity together with the tolerance it must be reproduced within + and the evidence behind that tolerance. Tolerances are derived from measured convergence + plateau drift and cross-platform spread — never widened to make a check pass. A failing + --check is fixed by explaining the physics or fixing the regression, not by loosening rtol. + Exit status: 0 all runs completed (and, with --fail-on-change, nothing changed) - 1 a run failed, or a quantity changed under --fail-on-change + 1 a run failed, a quantity changed under --fail-on-change, or a golden check failed Examples: # Compare two refs @@ -150,6 +175,13 @@ Examples: # Compare current working tree against develop julia --project=regression-harness regression-harness/regress.jl \\ --cases solovev_n1 --refs develop,local + + # Gate the working tree against the committed golden values + julia --project=regression-harness regression-harness/regress.jl --check --cases solovev_n1 + + # Regenerate goldens after an intended physics change + julia --project=regression-harness regression-harness/regress.jl \\ + --update-golden --cases solovev_n1 --reason "two-point b1 fix changes resonant field" """ function main(args=ARGS) @@ -189,14 +221,29 @@ function main(args=ARGS) [all_cases[name] for name in opts.cases if haskey(all_cases, name) || error("Unknown case: $name")] end - # Resolve refs + golden_mode = opts.check || opts.update_golden + if opts.check && opts.update_golden + error("--check and --update-golden are mutually exclusive: one asserts the goldens, the other rewrites them") + end + if opts.update_golden && (opts.reason === nothing || isempty(strip(opts.reason))) + error("--update-golden requires --reason \"...\": a golden change is a claim about physics, " * + "and the reviewer needs to know what changed and why") + end + + # Resolve refs. Golden modes judge the working tree, so they default to it rather than + # requiring the caller to spell out --refs local. resolved_refs = if opts.ref_range !== nothing expand_ref_range(opts.ref_range, REPO_ROOT) elseif !isempty(opts.refs) [resolve_ref(ref, REPO_ROOT) for ref in opts.refs] + elseif golden_mode + [resolve_ref(LOCAL_REF, REPO_ROOT)] else error("Must specify --refs or --ref-range") end + if golden_mode && length(resolved_refs) != 1 + error("Golden modes compare one ref against the committed values; got $(length(resolved_refs)) refs") + end if isempty(resolved_refs) error("No commits resolved from the given refs") @@ -220,6 +267,8 @@ function main(args=ARGS) n_failed = 0 n_changed = 0 + n_golden_fail = 0 + n_untracked = 0 # Run every case against each ref, grouped by ref so cases sharing a commit share # one worktree (and one Pkg.instantiate/precompile) instead of paying for it per case. @@ -236,20 +285,36 @@ function main(args=ARGS) println("Case: $(case_spec.name) — $(case_spec.description)") println("="^64) - summary = if length(resolved_refs) == 2 - report_two_ref_comparison(db, case_spec, - resolved_refs[1], resolved_refs[2]) + if opts.update_golden + update_golden_from_run(db, case_spec, resolved_refs[1].commit_hash, + String(opts.reason), REPO_ROOT) + elseif opts.check + summary = report_golden_check(db, case_spec, resolved_refs[1].commit_hash) + n_golden_fail += summary.n_fail + n_untracked += summary.n_untracked else - report_multi_ref(db, case_spec, resolved_refs) + summary = if length(resolved_refs) == 2 + report_two_ref_comparison(db, case_spec, + resolved_refs[1], resolved_refs[2]) + else + report_multi_ref(db, case_spec, resolved_refs) + end + n_failed += summary.n_failed + n_changed += summary.n_changed end - n_failed += summary.n_failed - n_changed += summary.n_changed end if n_failed > 0 @error "$n_failed run(s) failed — see the reports above" exit(1) end + if n_golden_fail > 0 + @error "$n_golden_fail quantity/quantities are outside their golden tolerance" + exit(1) + end + if n_untracked > 0 + @info "$n_untracked tracked quantity/quantities have no golden value yet (not gating)" + end if opts.fail_on_change && n_changed > 0 @error "$n_changed quantity/quantities changed (--fail-on-change)" exit(1) diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl new file mode 100644 index 000000000..7ca13da65 --- /dev/null +++ b/regression-harness/src/golden.jl @@ -0,0 +1,456 @@ +""" +Golden values: committed, reviewable reference numbers for a case. + +The `--refs` comparison answers "did this change?"; it cannot answer "is this right?", because +both sides come from the same working copy of the code and nothing is recorded in git. Golden +values close that gap: one TOML file per case, tracked alongside the source, holding the value +of every quantity plus the tolerance it must be reproduced within and the evidence behind that +tolerance. + +The tolerance policy is the load-bearing part. A tolerance is a claim about how much a quantity +may legitimately move, so it is *derived* — from the residual drift at a convergence plateau and +from the spread measured across platforms — never chosen to make a check pass. `update_golden` +refuses to write a tolerance below the recorded platform spread, and a failing check is fixed by +explaining the physics or fixing the regression, not by widening the bound. +""" + +""" +How strictly a quantity must reproduce, and why. + + - `topological` — integer counts and mode numbers; exact equality + - `equilibrium_scalar` — spline/quadrature outputs with no adaptive branching; very tight + - `physics_converged` — the quantities golden values exist for (δW, Δ′, torque, growth + rates); tolerance comes from the measured convergence plateau and platform spread + - `diagnostic` — ODE step counts, runtimes; recorded and reported, never gating, because they + describe the numerics rather than the physics + - `unconverged` — measured and found to have no plateau; tracked differentially, never pinned, + so that a known-unconverged quantity is visibly excluded instead of quietly given a wide bound +""" +const TOLERANCE_CLASSES = ("topological", "equilibrium_scalar", "physics_converged", "diagnostic", "unconverged") + +"""Classes whose failure fails the run. `diagnostic` and `unconverged` are reported only.""" +const GATING_CLASSES = ("topological", "equilibrium_scalar", "physics_converged") + +""" +Provisional per-class tolerances, used only when no measurement has been supplied. + +These are starting points, not results: a golden file written with them is stamped +`tolerance_basis = "class-default (provisional)"` so that an unmeasured pin is never mistaken +for a converged one. +""" +const CLASS_DEFAULT_TOLERANCE = Dict( + "topological" => (rtol=0.0, atol=0.0), + "equilibrium_scalar" => (rtol=1e-9, atol=0.0), + "physics_converged" => (rtol=1e-6, atol=0.0), + "diagnostic" => (rtol=Inf, atol=Inf), + "unconverged" => (rtol=Inf, atol=Inf) +) + +""" +One quantity's pinned value and the terms it is judged by. + +## Fields + + - `name` — quantity key, matching the case's `[quantities.*]` table + - `value_type` — "real", "integer", "json_array" (mirrors `ExtractedQuantity`) + - `value_real` / `value_int` / `value_text` — the pinned value in its stored form + - `rtol` / `atol` — pass when `|x - gold| <= atol + rtol*|gold|` + - `class` — one of `TOLERANCE_CLASSES` + - `tolerance_basis` — how the tolerance was arrived at: "measured" or + "class-default (provisional)" + - `plateau_drift` — relative movement over the final refinement of the convergence scan; NaN + when not measured + - `platform_spread` — relative disagreement between reference platforms; NaN when not measured + - `converged_at` — the discretization the value was taken at, e.g. + "grid_type=ldp, mpsi=1024, mtheta=512" +""" +struct GoldenValue + name::String + value_type::String + value_real::Union{Float64,Nothing} + value_int::Union{Int,Nothing} + value_text::Union{String,Nothing} + rtol::Float64 + atol::Float64 + class::String + tolerance_basis::String + plateau_drift::Float64 + platform_spread::Float64 + converged_at::String +end + +is_gating(g::GoldenValue) = g.class in GATING_CLASSES + +""" +Provenance for a whole golden file: what produced these numbers and why they last changed. +""" +struct GoldenMeta + case::String + golden_version::Int + generated_at::String + commit::String + reason::String + julia_version::String + os_arch::String + manifest_sha::String + nthreads::Int + blas_threads::Int +end + +golden_path(case_name::AbstractString) = joinpath(GOLDEN_DIR, "$(case_name).toml") + +has_golden(case_name::AbstractString) = isfile(golden_path(case_name)) + +""" +Read a golden file. Returns `(meta, Dict{name => GoldenValue})`, or `nothing` when absent. +""" +function load_golden(case_name::AbstractString) + path = golden_path(case_name) + isfile(path) || return nothing + data = TOML.parsefile(path) + m = get(data, "meta", Dict{String,Any}()) + meta = GoldenMeta( + get(m, "case", String(case_name)), + Int(get(m, "golden_version", 0)), + get(m, "generated_at", ""), + get(m, "commit", ""), + get(m, "reason", ""), + get(m, "julia_version", ""), + get(m, "os_arch", ""), + get(m, "manifest_sha", ""), + Int(get(m, "nthreads", -1)), + Int(get(m, "blas_threads", -1)) + ) + values = Dict{String,GoldenValue}() + for (name, v) in get(data, "values", Dict{String,Any}()) + class = get(v, "class", "physics_converged") + class in TOLERANCE_CLASSES || error("Golden file $path: quantity '$name' has unknown class '$class'") + values[name] = GoldenValue( + name, + get(v, "value_type", "real"), + haskey(v, "value") && v["value"] isa Real ? Float64(v["value"]) : nothing, + haskey(v, "value_int") ? Int(v["value_int"]) : nothing, + haskey(v, "value_text") ? String(v["value_text"]) : nothing, + Float64(get(v, "rtol", NaN)), + Float64(get(v, "atol", 0.0)), + class, + get(v, "tolerance_basis", "unspecified"), + Float64(get(v, "plateau_drift", NaN)), + Float64(get(v, "platform_spread", NaN)), + get(v, "converged_at", "") + ) + end + return (meta=meta, values=values) +end + +""" +Write a golden file. + +Refuses to emit a tolerance tighter than a recorded platform spread: a pin that no second +platform can reproduce is a broken gate, and silently loosening it later is exactly the failure +this whole mechanism exists to prevent. Raising the tolerance to cover a measured spread is a +deliberate act the caller performs, not something this function does behind the caller's back. +""" +function save_golden(meta::GoldenMeta, values::Dict{String,GoldenValue}) + for g in Base.values(values) + if isfinite(g.platform_spread) && isfinite(g.rtol) && g.rtol < g.platform_spread + error("Quantity '$(g.name)': rtol $(g.rtol) is tighter than the measured platform " * + "spread $(g.platform_spread). Widen it deliberately, with the measurement recorded, or " * + "reduce the spread before pinning.") + end + end + mkpath(GOLDEN_DIR) + out = Dict{String,Any}( + "meta" => Dict{String,Any}( + "case" => meta.case, + "golden_version" => meta.golden_version, + "generated_at" => meta.generated_at, + "commit" => meta.commit, + "reason" => meta.reason, + "julia_version" => meta.julia_version, + "os_arch" => meta.os_arch, + "manifest_sha" => meta.manifest_sha, + "nthreads" => meta.nthreads, + "blas_threads" => meta.blas_threads + ) + ) + vals = Dict{String,Any}() + for (name, g) in values + entry = Dict{String,Any}( + "value_type" => g.value_type, + "class" => g.class, + "tolerance_basis" => g.tolerance_basis, + "rtol" => g.rtol, + "atol" => g.atol + ) + g.value_real !== nothing && (entry["value"] = g.value_real) + g.value_int !== nothing && (entry["value_int"] = g.value_int) + g.value_text !== nothing && (entry["value_text"] = g.value_text) + isfinite(g.plateau_drift) && (entry["plateau_drift"] = g.plateau_drift) + isfinite(g.platform_spread) && (entry["platform_spread"] = g.platform_spread) + isempty(g.converged_at) || (entry["converged_at"] = g.converged_at) + vals[name] = entry + end + out["values"] = vals + open(golden_path(meta.case), "w") do io + TOML.print(io, out; sorted=true) + end + return golden_path(meta.case) +end + +""" +Classify a quantity from its case spec, so a new case gets sensible defaults without every +tolerance having to be written by hand. + +Integer counts are topological; runtimes and step counts describe the numerics rather than the +physics and so are diagnostic; everything else is assumed to be a physics quantity that must be +converged, which is the conservative assumption — it gates. +""" +function infer_class(spec::QuantitySpec)::String + spec.type == "runtime" && return "diagnostic" + startswith(spec.name, "nstep") && return "diagnostic" + spec.type == "int_scalar" && return "topological" + name = spec.name + equilibrium_names = ("q0", "q95", "betat", "betan", "betap1", "betap2", "betap3", "betaj", + "li1", "li2", "li3", "volume", "crnt", "bt0", "bwall", "aratio", "kappa", + "sing_psi", "sing_q") + name in equilibrium_names && return "equilibrium_scalar" + return "physics_converged" +end + +""" +Compare one extracted quantity against its golden value. + +Returns `(passed, deviation, detail)` where `deviation` is the relative deviation (absolute when +the golden value is zero) and `detail` is a human-readable reason on failure. Array quantities +pass only when every element is within tolerance; the reported deviation is the worst element. +""" +function compare_to_golden(q::NamedTuple, g::GoldenValue) + within = (x, gold) -> abs(x - gold) <= g.atol + g.rtol * abs(gold) + + if q.value_type != g.value_type + return (false, NaN, "type changed: golden $(g.value_type), got $(q.value_type)") + end + + if g.value_type == "integer" + (q.value_int === nothing || g.value_int === nothing) && return (false, NaN, "missing integer value") + d = Float64(abs(q.value_int - g.value_int)) + passed = g.class == "topological" ? q.value_int == g.value_int : within(Float64(q.value_int), Float64(g.value_int)) + return (passed, g.value_int == 0 ? d : d / abs(g.value_int), passed ? "" : "$(g.value_int) → $(q.value_int)") + + elseif g.value_type == "real" + (q.value_real === nothing || g.value_real === nothing) && return (false, NaN, "missing value") + gold = g.value_real + got = q.value_real + d = abs(got - gold) + rel = gold == 0.0 ? d : d / abs(gold) + return (within(got, gold), rel, within(got, gold) ? "" : @sprintf("%.9g → %.9g", gold, got)) + + elseif g.value_type == "json_array" + (q.value_text === nothing || g.value_text === nothing) && return (false, NaN, "missing array") + got = JSON.parse(q.value_text; allownan=true) + gold = JSON.parse(g.value_text; allownan=true) + length(got) == length(gold) && return _compare_arrays(got, gold, g, within) + return (false, NaN, "length $(length(gold)) → $(length(got))") + end + + return (false, NaN, "unsupported value type $(g.value_type)") +end + +"""Worst-element comparison for array quantities, shared by the real and complex encodings.""" +function _compare_arrays(got, gold, g::GoldenValue, within) + worst_rel = 0.0 + worst_idx = 0 + all_ok = true + for i in eachindex(gold) + a = _json_element_abs(gold[i]) + d = _json_element_diff(gold[i], got[i]) + rel = a == 0.0 ? d : d / a + ok = d <= g.atol + g.rtol * a + ok || (all_ok = false) + if rel > worst_rel + worst_rel = rel + worst_idx = i + end + end + detail = all_ok ? "" : "worst element $(worst_idx): rel $(@sprintf("%.3e", worst_rel))" + return (all_ok, worst_rel, detail) +end + +""" +Compare a case's fresh run against its golden file and print the report. + +Returns `(n_pass, n_fail, n_untracked, n_informational)`. Only gating classes can fail; +`diagnostic` and `unconverged` quantities are shown with their deviation and marked as +informational, so a reader sees them move without the run failing on them. +""" +function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String) + golden = load_golden(case_spec.name) + println() + println("Golden Check: $(case_spec.name)") + if golden === nothing + println(" No golden file at $(golden_path(case_spec.name)) — nothing to check against.") + println(" Generate one with: regress --update-golden --cases $(case_spec.name) --reason \"...\"") + return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0) + end + + quantities = get_quantities(db, commit_hash, case_spec.name) + info = get_run_info(db, commit_hash, case_spec.name) + + rows = Vector{Vector{String}}() + n_pass = n_fail = n_untracked = n_informational = 0 + + for spec in case_spec.quantities + g = get(golden.values, spec.name, nothing) + if g === nothing + spec.type == "runtime" && continue + n_untracked += 1 + continue + end + q = get(quantities, spec.name, nothing) + if q === nothing + push!(rows, [spec.label, g.class, "MISSING", "—", "FAIL"]) + n_fail += 1 + continue + end + passed, deviation, detail = compare_to_golden(q, g) + gating = is_gating(g) + status = if !gating + "info" + elseif passed + "ok" + else + "** FAIL **" + end + if !gating + n_informational += 1 + elseif passed + n_pass += 1 + else + n_fail += 1 + end + dev = isnan(deviation) ? "—" : @sprintf("%.2e", deviation) + tol = isfinite(g.rtol) ? @sprintf("%.1e", g.rtol) : "—" + push!(rows, [spec.label, g.class, dev, tol, status * (isempty(detail) ? "" : " $detail")]) + end + + header = ["Quantity", "Class", "Deviation", "rtol", "Status"] + widths = [length(h) for h in header] + for row in rows, i in eachindex(row) + widths[i] = max(widths[i], length(row[i])) + end + total_w = sum(widths) + 2 * (length(header) - 1) + + println("="^total_w) + println("Golden v$(golden.meta.golden_version), generated $(golden.meta.generated_at) @ $(golden.meta.commit)") + isempty(golden.meta.reason) || println("Reason: $(golden.meta.reason)") + println("Golden env: julia $(golden.meta.julia_version), $(golden.meta.os_arch), $(golden.meta.nthreads) thread/$(golden.meta.blas_threads) BLAS") + if info !== nothing && !isempty(info.fingerprint.julia_version) + println("This run: $(describe_env(info.fingerprint))") + info.fingerprint.os_arch == golden.meta.os_arch || + println(" NOTE: different platform from the one the goldens were measured on; deviations at or below the recorded platform_spread are expected.") + end + println("-"^total_w) + _print_row(header, widths) + println("-"^total_w) + for row in rows + _print_row(row, widths) + end + println("="^total_w) + parts = String[] + n_pass > 0 && push!(parts, "$n_pass pass") + n_fail > 0 && push!(parts, "$n_fail FAIL") + n_informational > 0 && push!(parts, "$n_informational informational") + n_untracked > 0 && push!(parts, "$n_untracked untracked") + println("Summary: ", join(parts, ", ")) + println() + return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational) +end + +""" +Regenerate a case's golden file from a completed run, reporting what moved. + +Prints an old→new delta for every quantity whose value changed, so the diff a reviewer sees in +git is accompanied by the size of each move. +""" +function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String, + reason::String, repo_root::String) + info = get_run_info(db, commit_hash, case_spec.name) + (info === nothing || !info.success) && error("Cannot update goldens for '$(case_spec.name)': the run did not succeed") + + # SQLite NULLs come back as `missing`, which the Union{...,Nothing} fields reject; `_column` + # normalizes both absent forms to nothing. + extracted = ExtractedQuantity[] + for (name, q) in get_quantities(db, commit_hash, case_spec.name) + push!(extracted, ExtractedQuantity(name, String(_column(q.label, name)), + _column(q.value_real, nothing), _column(q.value_int, nothing), + _column(q.value_text, nothing), q.value_type, q.noise_threshold)) + end + + previous = load_golden(case_spec.name) + existing = previous === nothing ? nothing : previous.values + values = build_golden_values(extracted, case_spec.quantities, existing) + + fp = info.fingerprint + meta = GoldenMeta(case_spec.name, + previous === nothing ? 1 : previous.meta.golden_version + 1, + Dates.format(Dates.now(), "yyyy-mm-dd"), + strip(read(`git -C $repo_root rev-parse --short HEAD`, String)), + reason, fp.julia_version, fp.os_arch, fp.manifest_sha, fp.nthreads, fp.blas_threads) + + println() + println("Golden update: $(case_spec.name) (v$(previous === nothing ? 0 : previous.meta.golden_version) → v$(meta.golden_version))") + if existing !== nothing + for (name, g) in sort(collect(values); by=first) + old = get(existing, name, nothing) + old === nothing && (println(@sprintf(" %-34s NEW", name)); continue) + if g.value_real !== nothing && old.value_real !== nothing && g.value_real != old.value_real + rel = old.value_real == 0 ? Inf : abs(g.value_real - old.value_real) / abs(old.value_real) + println(@sprintf(" %-34s %.9g → %.9g (rel %.2e)", name, old.value_real, g.value_real, rel)) + elseif g.value_text !== nothing && old.value_text !== nothing && g.value_text != old.value_text + println(@sprintf(" %-34s array changed", name)) + end + end + end + path = save_golden(meta, values) + println("Wrote $path ($(length(values)) quantities)") + provisional = count(g -> startswith(g.tolerance_basis, "class-default"), Base.values(values)) + provisional > 0 && println(" $provisional quantity/quantities still carry provisional class-default tolerances — " * + "replace them with measured plateau drift and platform spread before relying on this as a gate.") + println() + return path +end + +""" +Build golden entries from a set of freshly extracted quantities. + +`existing` carries forward the tolerance, class, and evidence already recorded for a quantity, so +regenerating values after a physics change does not silently reset hard-won measurements to +provisional defaults. +""" +function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector{QuantitySpec}, + existing::Union{Dict{String,GoldenValue},Nothing}) + spec_by_name = Dict(s.name => s for s in specs) + values = Dict{String,GoldenValue}() + for eq in extracted + eq.value_type == "missing" && continue + spec = get(spec_by_name, eq.name, nothing) + spec === nothing && continue + prior = existing === nothing ? nothing : get(existing, eq.name, nothing) + class = prior === nothing ? infer_class(spec) : prior.class + # Checksums have no notion of "close", so they cannot carry a tolerance; they stay a + # same-machine differential tool rather than a golden gate. + eq.value_type == "checksum" && continue + if prior === nothing + defaults = CLASS_DEFAULT_TOLERANCE[class] + rtol, atol, basis = defaults.rtol, defaults.atol, "class-default (provisional)" + drift, spread, at = NaN, NaN, "" + else + rtol, atol, basis = prior.rtol, prior.atol, prior.tolerance_basis + drift, spread, at = prior.plateau_drift, prior.platform_spread, prior.converged_at + end + values[eq.name] = GoldenValue(eq.name, eq.value_type, eq.value_real, eq.value_int, + eq.value_text, rtol, atol, class, basis, drift, spread, at) + end + return values +end diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl index 05c7159ce..0f5b85dd7 100644 --- a/regression-harness/src/types.jl +++ b/regression-harness/src/types.jl @@ -53,6 +53,11 @@ Parsed CLI options. worktree (pinning is on by default, so that two refs differ only by source code). `allow_env_mismatch` lets a cached result from a different environment be reused instead of re-run. `fail_on_change` turns any changed quantity into a non-zero exit status, for CI use. + +`check` runs the working tree and compares it against the committed golden values, which is the +mode CI gates on. `update_golden` regenerates those values from a fresh run; `reason` records +why, and is mandatory because a golden change is a claim about physics that a reviewer has to be +able to evaluate. """ struct CLIOptions cases::Vector{String} @@ -68,5 +73,8 @@ struct CLIOptions no_pin_manifest::Bool allow_env_mismatch::Bool fail_on_change::Bool + check::Bool + update_golden::Bool + reason::Union{String,Nothing} help::Bool end From fa669335e6f26e520ef6b127030fb99b18e24626 Mon Sep 17 00:00:00 2001 From: d-burg Date: Sat, 15 Aug 2026 17:16:22 -0400 Subject: [PATCH 03/30] REGRESSION - BUG FIX - Fail the golden check on orphaned golden entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check loop iterated only the case's quantities, so a golden entry whose quantity was renamed or dropped from the case TOML was never compared and never reported — the rename silently deleted its gate, with the new name appearing as merely "untracked". Found by adversarial review. Orphans now fail the check regardless of class, with a message pointing at golden regeneration for the reshaped case. Co-Authored-By: Claude Fable 5 --- regression-harness/src/golden.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 7ca13da65..2136fc860 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -334,6 +334,18 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St push!(rows, [spec.label, g.class, dev, tol, status * (isempty(detail) ? "" : " $detail")]) end + # Golden entries with no matching case quantity fail regardless of class: a renamed or + # removed quantity would otherwise silently delete its gate (the new name enters as merely + # "untracked"), and a check that quietly stops checking something is worse than one that + # fails. The fix is to regenerate the goldens for the reshaped case, with a reason. + spec_names = Set(spec.name for spec in case_spec.quantities) + for name in sort(collect(keys(golden.values))) + name in spec_names && continue + push!(rows, [name, golden.values[name].class, "ORPHANED", "—", + "** FAIL ** no matching quantity in the case — renamed or removed? Regenerate goldens."]) + n_fail += 1 + end + header = ["Quantity", "Class", "Deviation", "rtol", "Status"] widths = [length(h) for h in header] for row in rows, i in eachindex(row) From 2c5794515a9498b50d2cb60c34aabad6ecccfee2 Mon Sep 17 00:00:00 2001 From: d-burg Date: Sat, 15 Aug 2026 17:32:17 -0400 Subject: [PATCH 04/30] REGRESSION - BUG FIX - Harden the golden mechanism per adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifteen findings from a line-level adversarial review, the load-bearing ones: - A --check that finds no golden file for any requested case now exits 1: a green gate that checked nothing was indistinguishable from a passing one, so a deleted or typo-named golden file silently disabled CI. Mixed coverage warns per case. - --update-golden now prints a REMOVED line for every pin it drops: an h5 rename made extraction return missing, the entry silently vanished from the rewritten file, and the git diff read like intentional cleanup. Deleting a gate is now loud. - Golden provenance records "-dirty" when generated from an uncommitted tree; the commit field is what a reviewer uses to reproduce a disputed number, and it was recording HEAD for numbers HEAD cannot reproduce. - save_golden refuses valueless entries (a NaN scalar became a gating entry with a class and tolerance but no number — failing forever, with regeneration reproducing it identically). SLAYER gamma is legitimately NaN when no root is found, so this path is reachable. - The check path now normalizes SQLite missing before compare_to_golden (the update path already did); a NULL reaching the === nothing guards crashed the report with a non-boolean Missing TypeError instead of failing the quantity. - Tolerance validation runs on every load, not only at write: a hand edit or a merge taking the wrong side could otherwise ship a gating entry with no finite rtol (unsatisfiable) or an rtol below its recorded platform_spread (the exact quiet loosening this mechanism exists to prevent). - A value_type change resets carry-forward: a topological rtol=0 landing on a float gates at bit-exactness forever; a float tolerance landing on a count lets it drift. - A crashed run under --check reports as a crash, not as N tolerance failures. - Non-finite tolerances short-circuit within (Inf + NaN arithmetic made zero-valued diagnostic entries compare false); checksums no longer inflate the untracked count; sing_psi/sing_q reclassified physics_converged (root-found, not pure quadrature); nstep matched exactly instead of by prefix. Co-Authored-By: Claude Fable 5 --- regression-harness/regress.jl | 12 +++++++ regression-harness/src/golden.jl | 62 ++++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index f279bd6b6..371a69c7f 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -269,6 +269,8 @@ function main(args=ARGS) n_changed = 0 n_golden_fail = 0 n_untracked = 0 + n_checked = 0 + n_no_golden = 0 # Run every case against each ref, grouped by ref so cases sharing a commit share # one worktree (and one Pkg.instantiate/precompile) instead of paying for it per case. @@ -292,6 +294,7 @@ function main(args=ARGS) summary = report_golden_check(db, case_spec, resolved_refs[1].commit_hash) n_golden_fail += summary.n_fail n_untracked += summary.n_untracked + has_golden(case_spec.name) ? (n_checked += 1) : (n_no_golden += 1) else summary = if length(resolved_refs) == 2 report_two_ref_comparison(db, case_spec, @@ -312,6 +315,15 @@ function main(args=ARGS) @error "$n_golden_fail quantity/quantities are outside their golden tolerance" exit(1) end + if opts.check && n_checked == 0 + # A green gate that checked nothing is worse than a red one: a deleted or + # typo-named golden file must not be indistinguishable from a passing check. + @error "--check ran $n_no_golden case(s) but found no golden file for any of them — nothing was actually gated" + exit(1) + end + if opts.check && n_no_golden > 0 + @warn "$n_no_golden of $(n_no_golden + n_checked) requested case(s) have no golden file and were not gated" + end if n_untracked > 0 @info "$n_untracked tracked quantity/quantities have no golden value yet (not gating)" end diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 2136fc860..15db0ce53 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -125,6 +125,14 @@ function load_golden(case_name::AbstractString) for (name, v) in get(data, "values", Dict{String,Any}()) class = get(v, "class", "physics_converged") class in TOLERANCE_CLASSES || error("Golden file $path: quantity '$name' has unknown class '$class'") + # Enforced on every load, not only at write time: a hand edit or a merge taking the + # wrong side must not produce a gate quieter than the one save_golden refused to write. + if class in GATING_CLASSES + rt = Float64(get(v, "rtol", NaN)) + (isfinite(rt) && rt >= 0) || error("Golden file $path: gating quantity '$name' has no finite rtol — malformed or hand-edited entry") + sp = Float64(get(v, "platform_spread", NaN)) + isfinite(sp) && rt < sp && error("Golden file $path: quantity '$name' has rtol $rt below its recorded platform_spread $sp") + end values[name] = GoldenValue( name, get(v, "value_type", "real"), @@ -153,6 +161,11 @@ deliberate act the caller performs, not something this function does behind the """ function save_golden(meta::GoldenMeta, values::Dict{String,GoldenValue}) for g in Base.values(values) + if g.value_real === nothing && g.value_int === nothing && g.value_text === nothing + error("Quantity '$(g.name)': refusing to write a golden entry with no value (the run " * + "produced NaN or nothing) — a valueless gating entry fails forever and regenerating " * + "reproduces it. Exclude the quantity or fix the extraction.") + end if isfinite(g.platform_spread) && isfinite(g.rtol) && g.rtol < g.platform_spread error("Quantity '$(g.name)': rtol $(g.rtol) is tighter than the measured platform " * "spread $(g.platform_spread). Widen it deliberately, with the measurement recorded, or " * @@ -208,12 +221,13 @@ converged, which is the conservative assumption — it gates. """ function infer_class(spec::QuantitySpec)::String spec.type == "runtime" && return "diagnostic" - startswith(spec.name, "nstep") && return "diagnostic" + spec.name in ("nstep", "nstep_total") && return "diagnostic" spec.type == "int_scalar" && return "topological" name = spec.name + # sing_psi / sing_q are deliberately absent: singular-surface locations come from a root + # search, not pure spline/quadrature, so they take the measured physics_converged path. equilibrium_names = ("q0", "q95", "betat", "betan", "betap1", "betap2", "betap3", "betaj", - "li1", "li2", "li3", "volume", "crnt", "bt0", "bwall", "aratio", "kappa", - "sing_psi", "sing_q") + "li1", "li2", "li3", "volume", "crnt", "bt0", "bwall", "aratio", "kappa") name in equilibrium_names && return "equilibrium_scalar" return "physics_converged" end @@ -226,7 +240,10 @@ the golden value is zero) and `detail` is a human-readable reason on failure. Ar pass only when every element is within tolerance; the reported deviation is the worst element. """ function compare_to_golden(q::NamedTuple, g::GoldenValue) - within = (x, gold) -> abs(x - gold) <= g.atol + g.rtol * abs(gold) + # Non-finite tolerances mean "recorded, never judged" (diagnostic/unconverged); without this + # guard, gold == 0 turns atol + rtol*|gold| into Inf + NaN and the comparison is false. + within = (x, gold) -> !isfinite(g.atol) || !isfinite(g.rtol) || + abs(x - gold) <= g.atol + g.rtol * abs(gold) if q.value_type != g.value_type return (false, NaN, "type changed: golden $(g.value_type), got $(q.value_type)") @@ -296,6 +313,11 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St quantities = get_quantities(db, commit_hash, case_spec.name) info = get_run_info(db, commit_hash, case_spec.name) + if info !== nothing && !info.success + println(" RUN FAILED — nothing to compare (this is a crash, not a tolerance failure):") + println(" $(_short_err(info.error_msg))") + return (n_pass=0, n_fail=1, n_untracked=0, n_informational=0) + end rows = Vector{Vector{String}}() n_pass = n_fail = n_untracked = n_informational = 0 @@ -303,16 +325,23 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St for spec in case_spec.quantities g = get(golden.values, spec.name, nothing) if g === nothing - spec.type == "runtime" && continue + # Runtime and checksums are structurally un-goldenable (no tolerance semantics), so + # they must not inflate the untracked count that flags genuinely unpinned physics. + (spec.type == "runtime" || spec.extract == "checksum") && continue n_untracked += 1 continue end - q = get(quantities, spec.name, nothing) - if q === nothing + q_raw = get(quantities, spec.name, nothing) + if q_raw === nothing push!(rows, [spec.label, g.class, "MISSING", "—", "FAIL"]) n_fail += 1 continue end + # SQLite NULLs surface as , which the === nothing guards in compare_to_golden + # never match; normalize here as the update path already does. + q = (label=q_raw.label, value_real=_column(q_raw.value_real, nothing), + value_int=_column(q_raw.value_int, nothing), value_text=_column(q_raw.value_text, nothing), + value_type=q_raw.value_type, noise_threshold=q_raw.noise_threshold) passed, deviation, detail = compare_to_golden(q, g) gating = is_gating(g) status = if !gating @@ -409,10 +438,23 @@ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash: Dates.format(Dates.now(), "yyyy-mm-dd"), strip(read(`git -C $repo_root rev-parse --short HEAD`, String)), reason, fp.julia_version, fp.os_arch, fp.manifest_sha, fp.nthreads, fp.blas_threads) + if commit_hash == LOCAL_REF && !isempty(strip(read(`git -C $repo_root status --porcelain`, String))) + # The numbers came from uncommitted source; a clean HEAD checkout will not reproduce + # them, and the commit field is the one a reviewer uses to reproduce a disputed number. + meta = GoldenMeta(meta.case, meta.golden_version, meta.generated_at, + meta.commit * "-dirty", meta.reason, meta.julia_version, meta.os_arch, + meta.manifest_sha, meta.nthreads, meta.blas_threads) + @warn "Working tree is dirty: golden provenance recorded as $(meta.commit). Commit first if these values are meant to be reproducible." + end println() println("Golden update: $(case_spec.name) (v$(previous === nothing ? 0 : previous.meta.golden_version) → v$(meta.golden_version))") if existing !== nothing + for name in sort(collect(keys(existing))) + if !haskey(values, name) + println(@sprintf(" %-34s REMOVED — extraction returned missing (renamed h5 path?) or the quantity left the case. This deletes its gate; confirm it is intentional.", name)) + end + end for (name, g) in sort(collect(values); by=first) old = get(existing, name, nothing) old === nothing && (println(@sprintf(" %-34s NEW", name)); continue) @@ -449,6 +491,12 @@ function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector spec = get(spec_by_name, eq.name, nothing) spec === nothing && continue prior = existing === nothing ? nothing : get(existing, eq.name, nothing) + if prior !== nothing && prior.value_type != eq.value_type + # A type change invalidates the class and every measurement made under the old type; + # carrying a topological rtol=0 onto a float (or a float rtol onto a count) mis-gates. + @warn "Golden '$(eq.name)': value_type changed $(prior.value_type) → $(eq.value_type); resetting class and tolerances to provisional" + prior = nothing + end class = prior === nothing ? infer_class(spec) : prior.class # Checksums have no notion of "close", so they cannot carry a tolerance; they stay a # same-machine differential tool rather than a golden gate. From bab563a663d7a5cf1ed02dc0c45b2092c41a2ba0 Mon Sep 17 00:00:00 2001 From: d-burg Date: Sat, 15 Aug 2026 18:14:20 -0400 Subject: [PATCH 05/30] =?UTF-8?q?REGRESSION=20-=20IMPROVEMENT=20-=20Disamb?= =?UTF-8?q?iguate=20the=20=CE=94'=20variants;=20pin=20SLAYER=20inner-layer?= =?UTF-8?q?=20outputs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three distinct Δ' quantities exist and their labels no longer distinguish them poorly: - "Δ′ STRIDE-like BVP (PEST3)" — SingularSurfaces/Delta_prime_matrix from the riccati fundamental-matrix BVP; the tearing-stability Δ'. Pinned at source (riccati case) and now also as ingested by SLAYER, so a change in what SLAYER consumes is caught even when the source case is not run. - "Δ′ Galerkin RDCON (PEST3)" — the resistive-DCON Galerkin outer solution. Both outer solvers are arranged in the PEST3 convention; the labels now say so explicitly. - "PE driven-response Δ′ (SingularCoupling jump)" — the forced ∂b^ψ/∂ψ jump over 2π·χ₁ under applied flux: a driven-response diagnostic, not a tearing-stability Δ', and the case comment now says so. The per-surface ca-based Δ' (compute_delta_prime_from_ca!) is a stub — documented as not physically valid, never written to gpec.h5 — and is deliberately not pinned. The SLAYER case also gains the inner-layer solve's own per-surface outputs: complex layer thickness δ_s, δ_s/d_β, |δ_s| [m], and d_β [m]. The adaptive-scan Δ(Q) sample arrays are deliberately NOT pinned: AMR sample locations move under any refinement change, so they have no stable identity. Validated end-to-end: 21 quantities extract on the DIII-D SLAYER deck. Co-Authored-By: Claude Fable 5 --- .../cases/diiid_n1_riccati.toml | 4 +- regression-harness/cases/diiid_slayer_n1.toml | 46 +++++++++++++++++++ .../cases/gal_resistive_diiid.toml | 4 +- .../cases/gal_resistive_pe.toml | 6 ++- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/regression-harness/cases/diiid_n1_riccati.toml b/regression-harness/cases/diiid_n1_riccati.toml index f41d9ed0a..33e083765 100644 --- a/regression-harness/cases/diiid_n1_riccati.toml +++ b/regression-harness/cases/diiid_n1_riccati.toml @@ -16,7 +16,7 @@ example_dir = "examples/DIIID-like_riccati_deltaprime_example" h5path = "SingularSurfaces/Delta_prime_matrix" type = "complex_matrix" extract = "diagonal_complex" -label = "delta prime (BVP diagonal)" +label = "Δ′ STRIDE-like BVP (PEST3) diagonal" noise_threshold = 1e-8 order = 10 @@ -24,7 +24,7 @@ order = 10 h5path = "SingularSurfaces/Delta_prime_raw" type = "complex_matrix" extract = "all_complex" -label = "delta prime (raw side-major)" +label = "Δ′ STRIDE-like BVP (PEST3) raw side-major" noise_threshold = 1e-8 order = 11 diff --git a/regression-harness/cases/diiid_slayer_n1.toml b/regression-harness/cases/diiid_slayer_n1.toml index 90edf2e09..c34d89eb9 100644 --- a/regression-harness/cases/diiid_slayer_n1.toml +++ b/regression-harness/cases/diiid_slayer_n1.toml @@ -152,6 +152,52 @@ label = "SLAYER no_root flags [2/1,3/1,4/1]" noise_threshold = 0 order = 34 +# Outer-region Δ' consumed by the SLAYER matching — the same STRIDE-like BVP (PEST3) matrix +# the riccati case pins at source; tracked here too so a change in what SLAYER actually +# ingests is caught even if the source case is not run. +[quantities.slayer_delta_prime_diag] +h5path = "Tearing/PerSurface/Delta_prime_matrix" +type = "complex_matrix" +extract = "diagonal_complex" +label = "Δ′ STRIDE-like BVP (PEST3) diagonal, as ingested by SLAYER" +noise_threshold = 1e-8 +order = 40 + +# Inner-layer solve outputs per surface (the resistive-layer Riccati solution itself, not the +# outer matching). The adaptive-scan Δ(Q) sample arrays are deliberately NOT pinned: AMR sample +# locations move under any refinement change, so they have no stable identity to pin. +[quantities.slayer_delta_s] +h5path = "Tearing/LayerWidths/delta_s" +type = "complex_vector" +extract = "all_complex" +label = "SLAYER inner-layer thickness δ_s (complex, Riccati)" +noise_threshold = 1e-10 +order = 41 + +[quantities.slayer_delta_s_over_d_beta] +h5path = "Tearing/LayerWidths/delta_s_over_d_beta" +type = "complex_vector" +extract = "all_complex" +label = "SLAYER inner-layer δ_s/d_β (complex, dimensionless)" +noise_threshold = 1e-10 +order = 42 + +[quantities.slayer_delta_s_abs] +h5path = "Tearing/LayerWidths/delta_s_abs" +type = "real_vector" +extract = "all_real" +label = "SLAYER inner-layer physical thickness |δ_s| [m]" +noise_threshold = 1e-12 +order = 43 + +[quantities.slayer_d_beta] +h5path = "Tearing/LayerWidths/d_beta" +type = "real_vector" +extract = "all_real" +label = "SLAYER β-weighted ion drift scale d_β [m]" +noise_threshold = 1e-12 +order = 44 + # Settings (catches accidental config drift) [quantities.slayer_enabled] h5path = "Tearing/enabled" diff --git a/regression-harness/cases/gal_resistive_diiid.toml b/regression-harness/cases/gal_resistive_diiid.toml index b13388d8a..d028bc7e8 100644 --- a/regression-harness/cases/gal_resistive_diiid.toml +++ b/regression-harness/cases/gal_resistive_diiid.toml @@ -31,7 +31,7 @@ order = 11 h5path = "SingularSurfaces/Delta_prime_matrix" type = "complex_matrix" extract = "diagonal_complex" -label = "gal PEST3 Δ diagonal" +label = "Δ′ Galerkin RDCON (PEST3) diagonal" noise_threshold = 1e-6 order = 20 @@ -40,7 +40,7 @@ order = 20 h5path = "SingularSurfaces/Delta_prime_raw" type = "complex_matrix" extract = "norm" -label = "||gal Δ′ matrix||" +label = "||Δ′ Galerkin RDCON (PEST3) matrix||" noise_threshold = 1e-6 order = 21 diff --git a/regression-harness/cases/gal_resistive_pe.toml b/regression-harness/cases/gal_resistive_pe.toml index 9a04b30ac..322007eef 100644 --- a/regression-harness/cases/gal_resistive_pe.toml +++ b/regression-harness/cases/gal_resistive_pe.toml @@ -66,12 +66,14 @@ label = "penetrated b field" noise_threshold = 1e-8 order = 32 -# Per-surface Δ' from the PE singular-coupling analysis +# Per-surface driven-response Δ' from the PE singular-coupling jump (∂b^ψ/∂ψ across the +# rational surface over 2π·χ₁, forced by the applied flux). This is a driven-response +# diagnostic, NOT the tearing-stability Δ' of the STRIDE-like BVP or Galerkin RDCON solvers. [quantities.pe_delta_prime] h5path = "PerturbedEquilibrium/SingularCoupling/Delta_prime" type = "complex_vector" extract = "all_complex" -label = "PE Δ' per surface" +label = "PE driven-response Δ′ (SingularCoupling jump)" noise_threshold = 1e-8 order = 33 From 44c7f9e678ab3832f0992689b706971c446ba34c Mon Sep 17 00:00:00 2001 From: d-burg Date: Sat, 15 Aug 2026 20:13:25 -0400 Subject: [PATCH 06/30] =?UTF-8?q?REGRESSION=20-=20NEW=20FEATURE=20-=20Fixe?= =?UTF-8?q?d-Q=20probe=20of=20the=20SLAYER=20inner-layer=20=CE=94(Q)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AMR scan's Δ(Q) samples in gpec.h5 cannot carry golden values: adaptive sample locations move under any refinement change, so they have no stable identity to pin. This computed case pins the dispersion curve itself instead — Δ(Q) on a fixed 4×4 grid over Re(Q), Im(Q) ∈ [-10, 10], sixteen refinement-stable complex values. The layer parameters are the DIII-D-like SLAYER deck's own 2/1-surface values (extracted from Tearing/PerSurface of a deck run and quoted verbatim), so the case is self-contained and probes the solver alone; the parameter chain that produces those inputs is pinned separately by diiid_slayer_n1. Runs in ~1 s — fast-tier eligible alongside the GGJ references. All sixteen Δ values are finite across the grid, |Δ| spanning 0.21 to 1.52. Co-Authored-By: Claude Fable 5 --- .../cases/slayer_delta_probe.toml | 50 +++++++++++++++++++ regression-harness/src/runner.jl | 36 +++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 regression-harness/cases/slayer_delta_probe.toml diff --git a/regression-harness/cases/slayer_delta_probe.toml b/regression-harness/cases/slayer_delta_probe.toml new file mode 100644 index 000000000..41a7d8798 --- /dev/null +++ b/regression-harness/cases/slayer_delta_probe.toml @@ -0,0 +1,50 @@ +# Regression case: fixed-Q probe of the SLAYER inner-layer dispersion Δ(Q). +# Evaluates Δ(Q) on a fixed 4×4 grid over Re(Q), Im(Q) ∈ [-10, 10] using the DIII-D-like +# SLAYER deck's own 2/1-surface layer parameters, quoted verbatim in the runner's computed +# script. Fixed Q values are refinement-stable, unlike the AMR scan samples in gpec.h5, so +# this pins the dispersion curve itself; the parameter chain producing those layer inputs +# is pinned separately by diiid_slayer_n1. Seconds to run — fast-tier eligible. +[case] +name = "slayer_delta_probe" +description = "SLAYER inner-layer Δ(Q) on a fixed 4×4 Q grid, DIII-D-like 2/1 layer parameters" +kind = "computed" + +[quantities.q_re] +h5path = "slayer_probe/Q_re" +type = "real_vector" +extract = "all_real" +label = "probe grid Re(Q)" +noise_threshold = 0 +order = 10 + +[quantities.q_im] +h5path = "slayer_probe/Q_im" +type = "real_vector" +extract = "all_real" +label = "probe grid Im(Q)" +noise_threshold = 0 +order = 11 + +[quantities.delta_re] +h5path = "slayer_probe/Delta_re" +type = "real_vector" +extract = "all_real" +label = "SLAYER inner-layer Re Δ(Q) on the fixed grid" +noise_threshold = 1e-10 +order = 20 + +[quantities.delta_im] +h5path = "slayer_probe/Delta_im" +type = "real_vector" +extract = "all_real" +label = "SLAYER inner-layer Im Δ(Q) on the fixed grid" +noise_threshold = 1e-10 +order = 21 + +[quantities.runtime] +h5path = "" +type = "runtime" +extract = "value" +label = "Runtime (s)" +noise_threshold = 0.0 +order = 999 diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl index 1a192a140..eb8c601a8 100644 --- a/regression-harness/src/runner.jl +++ b/regression-harness/src/runner.jl @@ -123,6 +123,40 @@ end %RUNINFO% """ + +# Fixed-Q probe of the SLAYER inner-layer dispersion Δ(Q) on the DIII-D-like 2/1 surface. +# The adaptive AMR scan samples in gpec.h5 cannot be pinned (sample locations move under any +# refinement change), so this evaluates Δ(Q) on a fixed 4×4 grid over Re(Q), Im(Q) ∈ [-10, 10] +# instead — a refinement-stable pin of the dispersion curve itself. The layer parameters are the +# DIII-D-like SLAYER deck's own 2/1 surface values (Tearing/PerSurface, develop @ 1f193a6c), +# quoted verbatim so the case is self-contained and probes the SOLVER alone: the parameter +# chain that produces these numbers is pinned separately by the diiid_slayer_n1 case. +const COMPUTED_SLAYER_DELTA_PROBE_SCRIPT_TEMPLATE = """ +using Pkg +%INSTANTIATE% +using GeneralizedPerturbedEquilibrium +using GeneralizedPerturbedEquilibrium.InnerLayer +using HDF5 +p = SLAYERParameters(; + tau=1.1975430647804235, lu=6.086905739791344e6, c_beta=0.22094021004591707, + D_norm=4.191469284125091, P_perp=50.2972642329308, P_tor=34.74437891503841, + Q_e=1.0915286815773122, Q_i=-1.6558720999124832, iota_e=0.39729503206497013, + tauk=0.00010358784131467763, tau_r=3.453343933553279, delta_n=504.745127277822, + rs=0.3617373814196757, R0=1.7433359412007365, bt=1.0, sval_r=1.260093929519795, + eta=4.761642777337999e-8, d_beta=0.011249087118484661) +axis = range(-10.0, 10.0; length=4) +Q = ComplexF64[re + im_ * 1im for im_ in axis for re in axis] +t_start = time() +Δ = ComplexF64[solve_inner(SLAYERModel(), p, q).tearing for q in Q] +elapsed = time() - t_start +h5open(ARGS[1], "w") do fid + fid["slayer_probe/Q_re"] = real.(Q) + fid["slayer_probe/Q_im"] = imag.(Q) + fid["slayer_probe/Delta_re"] = real.(Δ) + fid["slayer_probe/Delta_im"] = imag.(Δ) +end +%RUNINFO% +""" # Self-contained separatrix-finder regression (PR #296). Loads a fixed-boundary EFIT whose # computational box hugs the LCFS (eps=0.05 TokaMaker aspect-scan g-file): outside the prescribed # LCFS the coil-vacuum flux turns back above the boundary value before the grid edge, so the old @@ -255,6 +289,8 @@ function _computed_script_template(case_spec::CaseSpec) return COMPUTED_GGJ_SCRIPT_TEMPLATE elseif case_spec.name == "ggj_ray_q500i" return COMPUTED_GGJ_RAY_SCRIPT_TEMPLATE + elseif case_spec.name == "slayer_delta_probe" + return COMPUTED_SLAYER_DELTA_PROBE_SCRIPT_TEMPLATE elseif case_spec.name == "efit_fixedbdy_separatrix" return COMPUTED_SEPARATRIX_SCRIPT_TEMPLATE end From cb0df351337ef458200431e28e053ed88c5d6189 Mon Sep 17 00:00:00 2001 From: d-burg Date: Mon, 17 Aug 2026 12:58:41 -0400 Subject: [PATCH 07/30] DOCS - BUG FIX - Backtick the chi_tor unit label so Documenter stops parsing it as a link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clarified chi_tor docstrings wrote "diffusivity [m²/s] (a viscosity ...)", which CommonMark reads as the link [m²/s](a viscosity ...). Documenter then fails the InnerLayer autodocs page with :cross_references, invalid local link/image. Backticking the unit is the convention CLAUDE.md prescribes for exactly this trap. Co-Authored-By: Claude Opus 5 --- src/InnerLayer/SLAYER/LayerInputs.jl | 2 +- src/InnerLayer/SLAYER/LayerParameters.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/InnerLayer/SLAYER/LayerInputs.jl b/src/InnerLayer/SLAYER/LayerInputs.jl index 92664c207..d42e7b190 100644 --- a/src/InnerLayer/SLAYER/LayerInputs.jl +++ b/src/InnerLayer/SLAYER/LayerInputs.jl @@ -184,7 +184,7 @@ profiles, without an intermediate file round-trip. - `chi_perp` -- perpendicular heat diffusivity [m²/s]. Scalar or a callable of `psi` (default `1.0`). - `chi_tor` -- Fitzpatrick's χ_φ: anomalous perpendicular ion MOMENTUM - diffusivity [m²/s] (a viscosity entering the vorticity and parallel-flow + diffusivity `[m²/s]` (a viscosity entering the vorticity and parallel-flow equations — not a heat diffusivity). Scalar or a callable of `psi` (default `1.0`). - `dr_val` -- resistive interchange index `D_R = E + F + H²` diff --git a/src/InnerLayer/SLAYER/LayerParameters.jl b/src/InnerLayer/SLAYER/LayerParameters.jl index a0ba39414..064cb6312 100644 --- a/src/InnerLayer/SLAYER/LayerParameters.jl +++ b/src/InnerLayer/SLAYER/LayerParameters.jl @@ -212,7 +212,7 @@ parametrization (P_perp/P_tor/D_norm; the older magnetic/electron Prandtl - `zeff` -- effective charge - `chi_perp` -- perpendicular energy diffusivity χ⊥ [m²/s] - `chi_tor` -- Fitzpatrick's χ_φ, the anomalous perpendicular ion momentum - diffusivity [m²/s] (a viscosity, not heat; TJ Layer.tex eqs. — enters the + diffusivity `[m²/s]` (a viscosity, not heat; TJ Layer.tex eqs. — enters the vorticity and parallel-flow equations, τ_φ = r_s²/χ_φ) - `m`, `n` -- poloidal / toroidal mode numbers at the surface - `dr_val`, `dgeo_val` -- inputs for the critical-Δ formula From bfdbde279ff3e956929ec3125898195ed15528f5 Mon Sep 17 00:00:00 2001 From: d-burg Date: Mon, 17 Aug 2026 13:24:44 -0400 Subject: [PATCH 08/30] REGRESSION - NEW FEATURE - Pin the declared integrator as Delta-prime provenance The Galerkin and STRIDE-like BVP Delta-prime are converging on one canonical SingularSurfaces/ path, so the numbers alone no longer say which formalism produced them: a deck switched from riccati to galerkin would be compared against the other formalism's gold with no signal. The schema forbids echoing control values into output groups, and the subgroup name that used to carry provenance is going away, so the pin reads the deck's own declaration out of the Input/gpec_toml_raw rerun snapshot -- the one group where control values belong. Adds a 'toml_key:' extraction yielding a 'token' value type, compared by exact equality and classed topological, since no tolerance means anything between "riccati" and "galerkin". A key the deck leaves unset pins as "", so starting or stopping declaring one is itself a change. Pinned on the four Delta-prime-bearing cases. The two Galerkin cases pin integrator AND gal_flag, because the Galerkin Delta-prime is currently the additive gal_flag path over a riccati sweep rather than a standalone integrator; when the standalone integrator lands and gal_flag retires, these fail loudly, which is the intent -- a deliberate formalism change must regenerate the gold. Co-Authored-By: Claude Opus 5 --- .../cases/diiid_n1_riccati.toml | 12 ++++++++++ regression-harness/cases/diiid_slayer_n1.toml | 11 ++++++++++ .../cases/gal_resistive_diiid.toml | 22 +++++++++++++++++++ .../cases/gal_resistive_pe.toml | 22 +++++++++++++++++++ regression-harness/src/extractor.jl | 20 ++++++++++++++++- regression-harness/src/golden.jl | 12 ++++++++++ regression-harness/src/types.jl | 5 +++-- 7 files changed, 101 insertions(+), 3 deletions(-) diff --git a/regression-harness/cases/diiid_n1_riccati.toml b/regression-harness/cases/diiid_n1_riccati.toml index 33e083765..811ed13c0 100644 --- a/regression-harness/cases/diiid_n1_riccati.toml +++ b/regression-harness/cases/diiid_n1_riccati.toml @@ -10,6 +10,18 @@ name = "diiid_n1_riccati" description = "DIII-D-like equilibrium, n=1, Riccati integrator Δ' matrix" example_dir = "examples/DIIID-like_riccati_deltaprime_example" +# Which formalism produced the Δ′ below. The Δ′ matrix is written to one canonical +# SingularSurfaces/ path whichever integrator computed it, so the numbers alone cannot say +# whether a gold pinned from the BVP is being compared against a Galerkin run. Pinning the +# deck's declared choice makes a formalism swap a loud failure instead of a silent re-gold. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 + # Tearing stability Δ' — the msing×msing BVP matrix diagonal, and the raw side-major # outer-region matrix it is projected from. [quantities.delta_prime] diff --git a/regression-harness/cases/diiid_slayer_n1.toml b/regression-harness/cases/diiid_slayer_n1.toml index c34d89eb9..6a1a4edf5 100644 --- a/regression-harness/cases/diiid_slayer_n1.toml +++ b/regression-harness/cases/diiid_slayer_n1.toml @@ -11,6 +11,17 @@ description = "DIII-D-like H-mode equilibrium, n=1, SLAYER tearing-mode analysis example_dir = "examples/DIIID-like_SLAYER_example" # Per-surface SLAYER layer parameters (geometry + dimensionless) +# Which formalism produced the Δ′ this case's layer physics consumes. The Δ′ matrix lands on +# one canonical SingularSurfaces/ path whichever integrator computed it, so pinning the deck's +# declared choice is what keeps a BVP-derived gold from being compared against a Galerkin run. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 + [quantities.slayer_ising] h5path = "Tearing/PerSurface/rational_index" type = "real_vector" diff --git a/regression-harness/cases/gal_resistive_diiid.toml b/regression-harness/cases/gal_resistive_diiid.toml index d028bc7e8..e2cf46fa6 100644 --- a/regression-harness/cases/gal_resistive_diiid.toml +++ b/regression-harness/cases/gal_resistive_diiid.toml @@ -10,6 +10,28 @@ description = "DIII-D-like, n=1, RDCON outer-region Galerkin Δ′ with rpec coi example_dir = "examples/DIIID-like_gal_resistive_example" # Number of resonant surfaces in the gal domain +# Which formalism produced the Galerkin Δ′ below. Both this deck's `integrator` and its +# `gal_flag` are pinned because the Galerkin Δ′ is currently the additive path (gal_flag +# layered on a riccati sweep) rather than a standalone integrator, so the pair is what names +# the formalism. Once the standalone `integrator = "galerkin"` lands and gal_flag is retired, +# these pins fail loudly — which is the intent: a deliberate formalism change must regenerate +# the gold rather than silently re-pointing it at different physics. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 + +[quantities.gal_flag] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.gal_flag" +label = "gal_flag (Δ′ provenance)" +noise_threshold = 0 +order = 2 + [quantities.gal_msing] h5path = "ForceFreeStates/Solutions/GalerkinIntegration/rational_count" type = "int_scalar" diff --git a/regression-harness/cases/gal_resistive_pe.toml b/regression-harness/cases/gal_resistive_pe.toml index 322007eef..ca54d455f 100644 --- a/regression-harness/cases/gal_resistive_pe.toml +++ b/regression-harness/cases/gal_resistive_pe.toml @@ -14,6 +14,28 @@ example_dir = "examples/DIIID-like_gal_resistive_pe_example" # any whole-profile ξ/ξ′ norm (those diverge at the rationals and are ill-posed as benchmarks). # Surface locations / count (sanity) +# Which formalism produced the Galerkin Δ′ below. Both this deck's `integrator` and its +# `gal_flag` are pinned because the Galerkin Δ′ is currently the additive path (gal_flag +# layered on a riccati sweep) rather than a standalone integrator, so the pair is what names +# the formalism. Once the standalone `integrator = "galerkin"` lands and gal_flag is retired, +# these pins fail loudly — which is the intent: a deliberate formalism change must regenerate +# the gold rather than silently re-pointing it at different physics. +[quantities.integrator] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.integrator" +label = "integrator (Δ′ provenance)" +noise_threshold = 0 +order = 1 + +[quantities.gal_flag] +h5path = "Input/gpec_toml_raw" +type = "token" +extract = "toml_key:ForceFreeStates.gal_flag" +label = "gal_flag (Δ′ provenance)" +noise_threshold = 0 +order = 2 + [quantities.pe_rational_q] h5path = "PerturbedEquilibrium/SingularCoupling/rational_q" type = "real_vector" diff --git a/regression-harness/src/extractor.jl b/regression-harness/src/extractor.jl index 33a328d7c..2d1d632f2 100644 --- a/regression-harness/src/extractor.jl +++ b/regression-harness/src/extractor.jl @@ -112,6 +112,24 @@ function apply_extraction(spec::QuantitySpec, raw)::ExtractedQuantity json_str = JSON.json(pairs; allownan=true) return ExtractedQuantity(name, label, nothing, nothing, json_str, "json_array", threshold) + elseif startswith(spec.extract, "toml_key:") + # "toml_key:": pin a deck's declared control value, read out of the + # `Input/gpec_toml_raw` rerun snapshot. Used to record which formalism a case asked + # for, because the Δ′ matrix is written to one canonical `SingularSurfaces/` path + # whichever integrator produced it, so the numbers alone no longer say. Reading it + # from `Input/` respects the schema rule that control values live nowhere else. + # A key the deck leaves unset pins as "", so a deck that starts or stops + # declaring one is itself a change rather than being silently absorbed. + keypath = spec.extract[(length("toml_key:")+1):end] + table = TOML.parse(raw isa AbstractString ? raw : String(raw)) + node = table + for key in split(keypath, ".") + node = (node isa AbstractDict && haskey(node, key)) ? node[key] : nothing + node === nothing && break + end + token = node === nothing ? "" : string(node) + return ExtractedQuantity(name, label, nothing, nothing, token, "token", threshold) + elseif spec.extract == "checksum" bytes = reinterpret(UInt8, vec(collect(raw))) hash = bytes2hex(sha256(bytes)) @@ -218,7 +236,7 @@ function compare_values(q1::NamedTuple, q2::NamedTuple) status = max_diff <= threshold ? "OK" : "CHANGED" return (max_diff, rel_diff, status) - elseif vtype == "checksum" + elseif vtype == "checksum" || vtype == "token" t1 = q1.value_text t2 = q2.value_text if t1 === nothing || t2 === nothing diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 15db0ce53..fb1dd1c30 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -223,6 +223,9 @@ function infer_class(spec::QuantitySpec)::String spec.type == "runtime" && return "diagnostic" spec.name in ("nstep", "nstep_total") && return "diagnostic" spec.type == "int_scalar" && return "topological" + # A declared control token is an exact-match gate, like a count: there is no tolerance + # that means anything between "riccati" and "galerkin". + startswith(spec.extract, "toml_key:") && return "topological" name = spec.name # sing_psi / sing_q are deliberately absent: singular-surface locations come from a root # search, not pure spline/quadrature, so they take the measured physics_converged path. @@ -269,6 +272,15 @@ function compare_to_golden(q::NamedTuple, g::GoldenValue) gold = JSON.parse(g.value_text; allownan=true) length(got) == length(gold) && return _compare_arrays(got, gold, g, within) return (false, NaN, "length $(length(gold)) → $(length(got))") + + elseif g.value_type == "token" + # A token names a discrete choice (which integrator produced the Δ′), so there is no + # tolerance to apply: it matches or the run is answering a different question than the + # gold does. Deviation is reported as 1.0 on mismatch rather than NaN so it sorts as a + # real failure in reports. + (q.value_text === nothing || g.value_text === nothing) && return (false, NaN, "missing token") + matched = q.value_text == g.value_text + return (matched, matched ? 0.0 : 1.0, matched ? "" : "$(g.value_text) → $(q.value_text)") end return (false, NaN, "unsupported value type $(g.value_type)") diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl index 0f5b85dd7..4c7393b06 100644 --- a/regression-harness/src/types.jl +++ b/regression-harness/src/types.jl @@ -8,8 +8,9 @@ Specification for a single quantity to extract from gpec.h5. struct QuantitySpec name::String h5path::String # HDF5 dataset path (e.g. "ForceFreeStates/FreeBoundaryStability/eigenmode_energies"), empty for runtime - type::String # "complex_vector", "real_vector", "real_scalar", "int_scalar", "real_matrix", "runtime" - extract::String # "value", "real_first", "imag_first", "abs_first", "norm", "all_real", "all_complex", "checksum" + type::String # "complex_vector", "real_vector", "real_scalar", "int_scalar", "real_matrix", "token", "runtime" + extract::String # "value", "real_first", "imag_first", "abs_first", "norm", "all_real", "all_complex", + # "diagonal_complex", "first_", "first__complex", "checksum", "toml_key:" label::String # Human-readable label for reports noise_threshold::Float64 # Absolute changes below this are noise order::Int # Display order in reports (lower = earlier) From fb8755b7b7a7bd8f9325f392b1848a8d8da16abc Mon Sep 17 00:00:00 2001 From: d-burg Date: Tue, 18 Aug 2026 13:47:05 -0400 Subject: [PATCH 09/30] Regression - MINOR - Drop the gal_flag provenance pin now that it is retired The Galerkin cases pinned both integrator and gal_flag because the Galerkin Delta-prime used to be the additive path -- gal_flag layered onto a riccati sweep -- so the pair was what named the formalism. The standalone galerkin integrator has since landed and gal_flag is gone from the control struct, so the decks now declare integrator = "galerkin" and that pin alone identifies the formalism. The gal_flag pin would only ever read "". Also reattaches two section comments that the original insertion orphaned onto the provenance block. Co-Authored-By: Claude Opus 5 --- .../cases/gal_resistive_diiid.toml | 20 +++++-------------- .../cases/gal_resistive_pe.toml | 20 +++++-------------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/regression-harness/cases/gal_resistive_diiid.toml b/regression-harness/cases/gal_resistive_diiid.toml index e2cf46fa6..bc0079848 100644 --- a/regression-harness/cases/gal_resistive_diiid.toml +++ b/regression-harness/cases/gal_resistive_diiid.toml @@ -9,13 +9,10 @@ name = "gal_resistive_diiid" description = "DIII-D-like, n=1, RDCON outer-region Galerkin Δ′ with rpec coil columns (delta_coil)" example_dir = "examples/DIIID-like_gal_resistive_example" -# Number of resonant surfaces in the gal domain -# Which formalism produced the Galerkin Δ′ below. Both this deck's `integrator` and its -# `gal_flag` are pinned because the Galerkin Δ′ is currently the additive path (gal_flag -# layered on a riccati sweep) rather than a standalone integrator, so the pair is what names -# the formalism. Once the standalone `integrator = "galerkin"` lands and gal_flag is retired, -# these pins fail loudly — which is the intent: a deliberate formalism change must regenerate -# the gold rather than silently re-pointing it at different physics. +# Which formalism produced the Galerkin Δ′ below. The Δ′ matrix is written to one canonical +# SingularSurfaces/ path whichever integrator computed it, so the numbers alone cannot say whether +# a gold pinned from the Galerkin solve is being compared against a BVP run. Pinning the deck's +# declared choice makes a formalism swap a loud failure instead of a silent re-gold. [quantities.integrator] h5path = "Input/gpec_toml_raw" type = "token" @@ -24,14 +21,7 @@ label = "integrator (Δ′ provenance)" noise_threshold = 0 order = 1 -[quantities.gal_flag] -h5path = "Input/gpec_toml_raw" -type = "token" -extract = "toml_key:ForceFreeStates.gal_flag" -label = "gal_flag (Δ′ provenance)" -noise_threshold = 0 -order = 2 - +# Number of resonant surfaces in the gal domain [quantities.gal_msing] h5path = "ForceFreeStates/Solutions/GalerkinIntegration/rational_count" type = "int_scalar" diff --git a/regression-harness/cases/gal_resistive_pe.toml b/regression-harness/cases/gal_resistive_pe.toml index ca54d455f..f943e8d49 100644 --- a/regression-harness/cases/gal_resistive_pe.toml +++ b/regression-harness/cases/gal_resistive_pe.toml @@ -13,13 +13,10 @@ example_dir = "examples/DIIID-like_gal_resistive_pe_example" # FINITE, per-rational-surface singular-coupling scalars (the physics-meaningful driven quantities) — not # any whole-profile ξ/ξ′ norm (those diverge at the rationals and are ill-posed as benchmarks). -# Surface locations / count (sanity) -# Which formalism produced the Galerkin Δ′ below. Both this deck's `integrator` and its -# `gal_flag` are pinned because the Galerkin Δ′ is currently the additive path (gal_flag -# layered on a riccati sweep) rather than a standalone integrator, so the pair is what names -# the formalism. Once the standalone `integrator = "galerkin"` lands and gal_flag is retired, -# these pins fail loudly — which is the intent: a deliberate formalism change must regenerate -# the gold rather than silently re-pointing it at different physics. +# Which formalism produced the Galerkin Δ′ below. The Δ′ matrix is written to one canonical +# SingularSurfaces/ path whichever integrator computed it, so the numbers alone cannot say whether +# a gold pinned from the Galerkin solve is being compared against a BVP run. Pinning the deck's +# declared choice makes a formalism swap a loud failure instead of a silent re-gold. [quantities.integrator] h5path = "Input/gpec_toml_raw" type = "token" @@ -28,14 +25,7 @@ label = "integrator (Δ′ provenance)" noise_threshold = 0 order = 1 -[quantities.gal_flag] -h5path = "Input/gpec_toml_raw" -type = "token" -extract = "toml_key:ForceFreeStates.gal_flag" -label = "gal_flag (Δ′ provenance)" -noise_threshold = 0 -order = 2 - +# Surface locations / count (sanity) [quantities.pe_rational_q] h5path = "PerturbedEquilibrium/SingularCoupling/rational_q" type = "real_vector" From f01c6a4ab23e96a4d72fe71267daffb586ac83de Mon Sep 17 00:00:00 2001 From: d-burg Date: Tue, 18 Aug 2026 19:28:26 -0400 Subject: [PATCH 10/30] Regression - FEATURE - Pin the del_s layer width against Fitzpatrick figs 6.2/6.3 Every other case in this suite pins GPEC against its own previous output, which answers "did this change?" but never "is this right?". This one pins it against published values: riccati_del_s integrates Fitzpatrick (IOP 2023) equations (6.29)/(6.30)/(6.38) directly and returns the plotted delta_s/d_beta, so prescribing the normalized parameters reproduces the figures' own axes. Verified at eight points inside the published contour bands, and the magnitudes match the book's table 6.2 reactor estimates at DIII-D-like parameters. Pure ODE -- no equilibrium, no geqdsk, no kinetic file -- so it is sub-second, fast-tier eligible, and carries no equilibrium dependence, meaning its golden values survive equilibrium-side re-baselining. Complements slayer_delta_probe, which pins the dispersion solve rather than the width solve. tau is not stated in the book's figure captions, so the reproduction's assumption (tau = 1, table 5.1) is pinned as a tracked quantity rather than left implicit. P_hat = 0 is excluded: it is a singular edge of the model, where alpha vanishes and F -> -i Q_hat at the origin, so pinning it would record a modelling boundary rather than a result. Co-Authored-By: Claude Opus 5 --- .../cases/slayer_dels_fitzpatrick.toml | 68 +++++++++++++++++++ regression-harness/src/runner.jl | 42 ++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 regression-harness/cases/slayer_dels_fitzpatrick.toml diff --git a/regression-harness/cases/slayer_dels_fitzpatrick.toml b/regression-harness/cases/slayer_dels_fitzpatrick.toml new file mode 100644 index 000000000..cc3c226bf --- /dev/null +++ b/regression-harness/cases/slayer_dels_fitzpatrick.toml @@ -0,0 +1,68 @@ +# Regression case: GPEC's del_s Riccati layer-width solver against Fitzpatrick, "Tearing Mode +# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3 (p. 6-10). +# +# This is the suite's only case pinned to an EXTERNAL published reference rather than to GPEC's own +# previous output: a failure means we no longer reproduce the textbook. Verified at eight points +# against the published contour bands, e.g. (Q,P) = (4.0, 0.5) -> Re 2.008 against the 1.9-2.1 +# contours, (0.5, 4.0) -> Re 1.537 against the 1.5 contour. The magnitudes also match the book's +# table 6.2 reactor estimates (|delta_s| 2.95-4.69 mm) at DIII-D-like parameters. +# +# Pure ODE: no equilibrium, no geqdsk, no kinetic file. Sub-second, so it is fast-tier eligible, and +# it carries no equilibrium dependence -- its golden values survive equilibrium-side re-baselining. +# Complements slayer_delta_probe, which pins the dispersion solve Delta(Q); this pins the width +# solve delta_s/d_beta over (Q_hat, P_hat). +[case] +name = "slayer_dels_fitzpatrick" +description = "del_s Riccati layer width vs Fitzpatrick 2023 figs 6.2/6.3 over (Q_hat, P_hat)" +kind = "computed" + +# The scan axes, pinned so a silently reshaped grid is a failure rather than a re-gold. +[quantities.q_hat] +h5path = "fitzpatrick/Q_hat" +type = "real_vector" +extract = "all_real" +label = "grid Q_hat (= Q_*/D^4)" +noise_threshold = 0 +order = 10 + +[quantities.p_hat] +h5path = "fitzpatrick/P_hat" +type = "real_vector" +extract = "all_real" +label = "grid P_hat (= P/D^6)" +noise_threshold = 0 +order = 11 + +# tau is not stated in the book's figure captions; tau = 1 (table 5.1, low-field reactor) is the +# assumption this reproduction rests on, so it is pinned rather than left implicit. +[quantities.tau] +h5path = "fitzpatrick/tau" +type = "real_scalar" +extract = "value" +label = "tau (assumed; not given in the figure captions)" +noise_threshold = 0 +order = 12 + +[quantities.dels_db_re] +h5path = "fitzpatrick/dels_db_re" +type = "real_vector" +extract = "all_real" +label = "Re(delta_s/d_beta) — Fitzpatrick fig 6.2" +noise_threshold = 1e-10 +order = 20 + +[quantities.dels_db_im] +h5path = "fitzpatrick/dels_db_im" +type = "real_vector" +extract = "all_real" +label = "Im(delta_s/d_beta) — Fitzpatrick fig 6.3" +noise_threshold = 1e-10 +order = 21 + +[quantities.runtime] +h5path = "" +type = "runtime" +extract = "value" +label = "Runtime (s)" +noise_threshold = 0.0 +order = 999 diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl index eb8c601a8..62dea2c60 100644 --- a/regression-harness/src/runner.jl +++ b/regression-harness/src/runner.jl @@ -157,6 +157,46 @@ h5open(ARGS[1], "w") do fid end %RUNINFO% """ +# External-reference validation: GPEC's del_s Riccati solver against Fitzpatrick, "Tearing Mode +# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3. Unlike every other case in this +# suite -- which pins GPEC against its own previous output -- this one pins it against published +# values, so a failure means "we no longer reproduce the textbook", not "we changed". +# +# riccati_del_s integrates the book's layer equations directly: E and F are (6.29)/(6.30), the +# Riccati form is (6.38), and the returned dels_db is the plotted delta_s/d_beta. Prescribing the +# normalized parameters (D_norm = 1, P_perp = P_tor = Phat, Q_e = Qhat/(1+1/tau)) makes the solver's +# internal Q_hat equal the book's Qhat_*, so the grid below is exactly the figures' axes. +# +# tau is NOT stated in the figure captions; tau = 1 (table 5.1, low-field reactor) is pinned here as +# an explicit assumption of the case, not inherited from a default that could drift. +# +# Phat = 0 is a singular edge of the model, not a hard case: alpha = sqrt(Phat_perp/(1+1/tau)) +# vanishes so the large-q boundary form degenerates, and F -> -i*Qhat at the origin. The grid starts +# just inside so the pins record results rather than a modelling boundary. +const COMPUTED_SLAYER_DELS_FITZPATRICK_SCRIPT_TEMPLATE = """ +using Pkg +%INSTANTIATE% +using GeneralizedPerturbedEquilibrium +using GeneralizedPerturbedEquilibrium.InnerLayer +using HDF5 +const TAU = 1.0 +probe(Q, P) = SLAYERParameters(; tau=TAU, lu=1.0, c_beta=0.0, D_norm=1.0, + P_perp=P, P_tor=P, Q_e=Q / (1 + 1/TAU), Q_i=0.0, iota_e=0.0, tauk=1.0, + tau_r=1.0, delta_n=1.0, rs=1.0, R0=1.0, bt=1.0, sval_r=1.0, eta=1.0, d_beta=1.0) +axis = [0.02, 0.5, 1.0, 2.0, 4.0] +QP = [(q, p) for p in axis for q in axis] +t_start = time() +dels = ComplexF64[riccati_del_s(probe(q, p)) for (q, p) in QP] +elapsed = time() - t_start +h5open(ARGS[1], "w") do fid + fid["fitzpatrick/Q_hat"] = Float64[q for (q, _) in QP] + fid["fitzpatrick/P_hat"] = Float64[p for (_, p) in QP] + fid["fitzpatrick/dels_db_re"] = real.(dels) + fid["fitzpatrick/dels_db_im"] = imag.(dels) + fid["fitzpatrick/tau"] = TAU +end +%RUNINFO% +""" # Self-contained separatrix-finder regression (PR #296). Loads a fixed-boundary EFIT whose # computational box hugs the LCFS (eps=0.05 TokaMaker aspect-scan g-file): outside the prescribed # LCFS the coil-vacuum flux turns back above the boundary value before the grid edge, so the old @@ -293,6 +333,8 @@ function _computed_script_template(case_spec::CaseSpec) return COMPUTED_SLAYER_DELTA_PROBE_SCRIPT_TEMPLATE elseif case_spec.name == "efit_fixedbdy_separatrix" return COMPUTED_SEPARATRIX_SCRIPT_TEMPLATE + elseif case_spec.name == "slayer_dels_fitzpatrick" + return COMPUTED_SLAYER_DELS_FITZPATRICK_SCRIPT_TEMPLATE end error("No computed-script template registered for case '$(case_spec.name)'") end From ce422f04eab9551aa384c781f44048fda132d802 Mon Sep 17 00:00:00 2001 From: d-burg Date: Tue, 18 Aug 2026 19:35:54 -0400 Subject: [PATCH 11/30] Regression - MINOR - Golden values for the Fitzpatrick del_s reproduction Generated at af3e4492 on a clean tree. Tolerances are the provisional class defaults: this is a deterministic ODE with no equilibrium input, so the measured platform spread should replace them before it is relied on as a gate. Co-Authored-By: Claude Opus 5 --- .../golden/slayer_dels_fitzpatrick.toml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 regression-harness/golden/slayer_dels_fitzpatrick.toml diff --git a/regression-harness/golden/slayer_dels_fitzpatrick.toml b/regression-harness/golden/slayer_dels_fitzpatrick.toml new file mode 100644 index 000000000..ddb9ce848 --- /dev/null +++ b/regression-harness/golden/slayer_dels_fitzpatrick.toml @@ -0,0 +1,59 @@ +[meta] +blas_threads = 2 +case = "slayer_dels_fitzpatrick" +commit = "af3e4492" +generated_at = "2026-08-18" +golden_version = 1 +julia_version = "1.11.6" +manifest_sha = "7e5c34ad1722fa6addf94f7387b972fa8ac17ff86166793819b158496d495132" +nthreads = 2 +os_arch = "arm64-apple-darwin24.0.0" +reason = "Initial pin: validated against Fitzpatrick 2023 figs 6.2/6.3 at eight contour points" + +[values.dels_db_im] +atol = 0.0 +class = "physics_converged" +rtol = 1.0e-6 +tolerance_basis = "class-default (provisional)" +value_text = "[-0.03903669609227284,-0.7097932061530505,-1.0865009301031268,-1.5690999603536648,-2.2242340056733148,-0.011118192607611124,-0.35424388489008746,-0.776484631542084,-1.416939644030615,-2.2250603072320665,-0.008402000357094586,-0.23928032159546678,-0.5666784684880879,-1.241924594739638,-2.1750391797697133,-0.006297613829213364,-0.1657149360631757,-0.3716832232708168,-0.9092532037784165,-1.9732264855758332,-0.004682490543220957,-0.11916296976842101,-0.2502562086693129,-0.578522722631012,-1.4658508790642786]" +value_type = "json_array" + +[values.dels_db_re] +atol = 0.0 +class = "physics_converged" +rtol = 1.0e-6 +tolerance_basis = "class-default (provisional)" +value_text = "[0.46252746222290736,0.7691732891864712,1.0821020639186198,1.5537655127977916,2.2130417549691974,0.9838377906344368,0.9426353501706125,1.017567466663986,1.35072260354625,2.0080394342246684,1.1484429925403958,1.1069495858506195,1.0754748077370269,1.2328881423647,1.8273926389542299,1.336215433387759,1.3118923623802783,1.259193827055807,1.2120466712964955,1.540405108811276,1.5494641775774003,1.5373751336175125,1.5040680534472435,1.4084024975856595,1.350103015944749]" +value_type = "json_array" + +[values.p_hat] +atol = 0.0 +class = "physics_converged" +rtol = 1.0e-6 +tolerance_basis = "class-default (provisional)" +value_text = "[0.02,0.02,0.02,0.02,0.02,0.5,0.5,0.5,0.5,0.5,1.0,1.0,1.0,1.0,1.0,2.0,2.0,2.0,2.0,2.0,4.0,4.0,4.0,4.0,4.0]" +value_type = "json_array" + +[values.q_hat] +atol = 0.0 +class = "physics_converged" +rtol = 1.0e-6 +tolerance_basis = "class-default (provisional)" +value_text = "[0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0]" +value_type = "json_array" + +[values.runtime] +atol = +inf +class = "diagnostic" +rtol = +inf +tolerance_basis = "class-default (provisional)" +value = 1.2057960033416748 +value_type = "real" + +[values.tau] +atol = 0.0 +class = "physics_converged" +rtol = 1.0e-6 +tolerance_basis = "class-default (provisional)" +value = 1.0 +value_type = "real" From 25d68260949f3bdc6308e262893d26f95e642f46 Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 28 Aug 2026 13:23:34 -0400 Subject: [PATCH 12/30] Regression - BUGFIX - Guard non-finite tolerances in the golden array comparison Mirror the scalar path's non-finite guard in the array comparison (a diagnostic-class array with a zero gold element hit Inf*0 = NaN and was marked failing), drop the now-unused closure argument, and restore the dropped word in the SQLite-NULL comment. Co-Authored-By: Claude Fable 5 --- regression-harness/src/golden.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index fb1dd1c30..4c44c6086 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -270,7 +270,7 @@ function compare_to_golden(q::NamedTuple, g::GoldenValue) (q.value_text === nothing || g.value_text === nothing) && return (false, NaN, "missing array") got = JSON.parse(q.value_text; allownan=true) gold = JSON.parse(g.value_text; allownan=true) - length(got) == length(gold) && return _compare_arrays(got, gold, g, within) + length(got) == length(gold) && return _compare_arrays(got, gold, g) return (false, NaN, "length $(length(gold)) → $(length(got))") elseif g.value_type == "token" @@ -287,7 +287,7 @@ function compare_to_golden(q::NamedTuple, g::GoldenValue) end """Worst-element comparison for array quantities, shared by the real and complex encodings.""" -function _compare_arrays(got, gold, g::GoldenValue, within) +function _compare_arrays(got, gold, g::GoldenValue) worst_rel = 0.0 worst_idx = 0 all_ok = true @@ -295,7 +295,9 @@ function _compare_arrays(got, gold, g::GoldenValue, within) a = _json_element_abs(gold[i]) d = _json_element_diff(gold[i], got[i]) rel = a == 0.0 ? d : d / a - ok = d <= g.atol + g.rtol * a + # Same non-finite guard as the scalar path: Inf tolerances mean "recorded, never + # judged", and Inf*0 = NaN would otherwise mark a zero gold element as failing. + ok = !isfinite(g.atol) || !isfinite(g.rtol) || d <= g.atol + g.rtol * a ok || (all_ok = false) if rel > worst_rel worst_rel = rel @@ -349,7 +351,7 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St n_fail += 1 continue end - # SQLite NULLs surface as , which the === nothing guards in compare_to_golden + # SQLite NULLs surface as `missing`, which the === nothing guards in compare_to_golden # never match; normalize here as the update path already does. q = (label=q_raw.label, value_real=_column(q_raw.value_real, nothing), value_int=_column(q_raw.value_int, nothing), value_text=_column(q_raw.value_text, nothing), From 41debc2eb2cd338238c0f678317a2c1ca9a73a02 Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 28 Aug 2026 13:23:34 -0400 Subject: [PATCH 13/30] Regression - MINOR - Tighten the computed-case provenance comments Flag bt as a placeholder the dispersion solve never reads, and point the Fitzpatrick validation evidence at the case TOML instead of duplicating it. Co-Authored-By: Claude Fable 5 --- regression-harness/src/runner.jl | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl index 62dea2c60..f59ddec60 100644 --- a/regression-harness/src/runner.jl +++ b/regression-harness/src/runner.jl @@ -128,9 +128,9 @@ end # The adaptive AMR scan samples in gpec.h5 cannot be pinned (sample locations move under any # refinement change), so this evaluates Δ(Q) on a fixed 4×4 grid over Re(Q), Im(Q) ∈ [-10, 10] # instead — a refinement-stable pin of the dispersion curve itself. The layer parameters are the -# DIII-D-like SLAYER deck's own 2/1 surface values (Tearing/PerSurface, develop @ 1f193a6c), -# quoted verbatim so the case is self-contained and probes the SOLVER alone: the parameter -# chain that produces these numbers is pinned separately by the diiid_slayer_n1 case. +# DIII-D-like SLAYER deck's own 2/1 surface values (Tearing/PerSurface), quoted so the case is +# self-contained and probes the SOLVER alone (bt is a placeholder — the dispersion solve never +# reads it); the parameter chain producing these numbers is pinned separately by diiid_slayer_n1. const COMPUTED_SLAYER_DELTA_PROBE_SCRIPT_TEMPLATE = """ using Pkg %INSTANTIATE% @@ -158,21 +158,12 @@ end %RUNINFO% """ # External-reference validation: GPEC's del_s Riccati solver against Fitzpatrick, "Tearing Mode -# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3. Unlike every other case in this -# suite -- which pins GPEC against its own previous output -- this one pins it against published -# values, so a failure means "we no longer reproduce the textbook", not "we changed". -# -# riccati_del_s integrates the book's layer equations directly: E and F are (6.29)/(6.30), the -# Riccati form is (6.38), and the returned dels_db is the plotted delta_s/d_beta. Prescribing the -# normalized parameters (D_norm = 1, P_perp = P_tor = Phat, Q_e = Qhat/(1+1/tau)) makes the solver's -# internal Q_hat equal the book's Qhat_*, so the grid below is exactly the figures' axes. -# -# tau is NOT stated in the figure captions; tau = 1 (table 5.1, low-field reactor) is pinned here as -# an explicit assumption of the case, not inherited from a default that could drift. -# -# Phat = 0 is a singular edge of the model, not a hard case: alpha = sqrt(Phat_perp/(1+1/tau)) -# vanishes so the large-q boundary form degenerates, and F -> -i*Qhat at the origin. The grid starts -# just inside so the pins record results rather than a modelling boundary. +# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3 — see the case TOML header for +# the validation evidence. Prescribing the normalized parameters (D_norm = 1, +# P_perp = P_tor = Phat, Q_e = Qhat/(1+1/tau)) makes the solver's internal Q_hat equal the +# book's Qhat_*, so the grid below is exactly the figures' axes. tau = 1 is an explicit pinned +# assumption (not stated in the captions). The grid starts just inside Phat = 0, which is a +# singular edge of the model (alpha vanishes and the large-q boundary form degenerates). const COMPUTED_SLAYER_DELS_FITZPATRICK_SCRIPT_TEMPLATE = """ using Pkg %INSTANTIATE% From e4131da1d2f67eeedd0ab85f12dc2075f81170a4 Mon Sep 17 00:00:00 2001 From: d-burg Date: Mon, 31 Aug 2026 15:45:45 -0400 Subject: [PATCH 14/30] Regression - BUGFIX - Report a crashed golden-check run as a crash, not a tolerance failure A case that crashed before producing results was counted in n_fail and exited as 'outside their golden tolerance'. The crash now travels in its own n_run_failed field and gets a crash-specific exit message. Co-Authored-By: Claude Fable 5 --- regression-harness/regress.jl | 6 ++++++ regression-harness/src/golden.jl | 14 ++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 371a69c7f..7b823ed86 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -268,6 +268,7 @@ function main(args=ARGS) n_failed = 0 n_changed = 0 n_golden_fail = 0 + n_golden_crashed = 0 n_untracked = 0 n_checked = 0 n_no_golden = 0 @@ -293,6 +294,7 @@ function main(args=ARGS) elseif opts.check summary = report_golden_check(db, case_spec, resolved_refs[1].commit_hash) n_golden_fail += summary.n_fail + n_golden_crashed += summary.n_run_failed n_untracked += summary.n_untracked has_golden(case_spec.name) ? (n_checked += 1) : (n_no_golden += 1) else @@ -311,6 +313,10 @@ function main(args=ARGS) @error "$n_failed run(s) failed — see the reports above" exit(1) end + if n_golden_crashed > 0 + @error "$n_golden_crashed case(s) crashed before producing results — see the reports above (a crash, not a tolerance failure)" + exit(1) + end if n_golden_fail > 0 @error "$n_golden_fail quantity/quantities are outside their golden tolerance" exit(1) diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 4c44c6086..94170f1d8 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -311,9 +311,11 @@ end """ Compare a case's fresh run against its golden file and print the report. -Returns `(n_pass, n_fail, n_untracked, n_informational)`. Only gating classes can fail; -`diagnostic` and `unconverged` quantities are shown with their deviation and marked as -informational, so a reader sees them move without the run failing on them. +Returns `(n_pass, n_fail, n_untracked, n_informational, n_run_failed)`. Only gating classes +can fail; `diagnostic` and `unconverged` quantities are shown with their deviation and marked +as informational, so a reader sees them move without the run failing on them. A run that +crashed outright is counted in `n_run_failed`, never in `n_fail`, so the caller can exit with +a crash message instead of a tolerance message. """ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String) golden = load_golden(case_spec.name) @@ -322,7 +324,7 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St if golden === nothing println(" No golden file at $(golden_path(case_spec.name)) — nothing to check against.") println(" Generate one with: regress --update-golden --cases $(case_spec.name) --reason \"...\"") - return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0) + return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=0) end quantities = get_quantities(db, commit_hash, case_spec.name) @@ -330,7 +332,7 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St if info !== nothing && !info.success println(" RUN FAILED — nothing to compare (this is a crash, not a tolerance failure):") println(" $(_short_err(info.error_msg))") - return (n_pass=0, n_fail=1, n_untracked=0, n_informational=0) + return (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=1) end rows = Vector{Vector{String}}() @@ -419,7 +421,7 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St n_untracked > 0 && push!(parts, "$n_untracked untracked") println("Summary: ", join(parts, ", ")) println() - return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational) + return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational, n_run_failed=0) end """ From cc13288edf75a0de0430c21375c32ee2f624b5cd Mon Sep 17 00:00:00 2001 From: d-burg Date: Mon, 31 Aug 2026 16:04:15 -0400 Subject: [PATCH 15/30] Regression - TEST - Unit-test the golden comparison logic on synthetic data Covers the comparison paths (scalar/integer/token/array with worst-element reporting and the non-finite-tolerance guard), type-change handling, the load/save validation refusals, infer_class, and the --check counting contract including crash-vs-tolerance separation. GOLDEN_DIR is redirected to a temp dir so the tracked pins are never touched. Co-Authored-By: Claude Fable 5 --- regression-harness/test/runtests.jl | 282 ++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 regression-harness/test/runtests.jl diff --git a/regression-harness/test/runtests.jl b/regression-harness/test/runtests.jl new file mode 100644 index 000000000..86cde0a18 --- /dev/null +++ b/regression-harness/test/runtests.jl @@ -0,0 +1,282 @@ +# Unit tests for the golden-value comparison logic (src/golden.jl), on synthetic data only — +# no GPEC runs. Run against the harness project, not the GPEC package: +# +# julia --project=regression-harness regression-harness/test/runtests.jl +# +using Test +using Dates, HDF5, JSON, Printf, SHA, SQLite, Tables, TOML + +const HARNESS_DIR = abspath(joinpath(@__DIR__, "..")) +const REPO_ROOT = abspath(joinpath(HARNESS_DIR, "..")) +const DEFAULT_DB_PATH = joinpath(mktempdir(), "test_cache.sqlite") +const CASES_DIR = joinpath(HARNESS_DIR, "cases") +# Redirected to a temp dir so tests can write and corrupt golden files freely without ever +# touching the tracked pins under regression-harness/golden/. +const GOLDEN_DIR = mktempdir() + +for f in ("types.jl", "env.jl", "config.jl", "database.jl", "utils.jl", "extractor.jl", "runner.jl", "reporter.jl", "golden.jl") + include(joinpath(HARNESS_DIR, "src", f)) +end + +qspec(name; type="real_scalar", extract="value", label=name, noise=0.0, order=1) = + QuantitySpec(name, "", type, extract, label, noise, order) + +golden_val(name; value_type="real", value_real=nothing, value_int=nothing, value_text=nothing, + rtol=1e-6, atol=0.0, class="physics_converged", basis="measured", drift=NaN, spread=NaN, at="") = + GoldenValue(name, value_type, value_real, value_int, value_text, rtol, atol, class, basis, drift, spread, at) + +qtuple(; value_real=nothing, value_int=nothing, value_text=nothing, value_type="real", label="q") = + (label=label, value_real=value_real, value_int=value_int, value_text=value_text, value_type=value_type, noise_threshold=0.0) + +extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, value_type="real") = + ExtractedQuantity(name, name, value_real, value_int, value_text, value_type, 0.0) + +@testset "golden comparison logic" begin + + @testset "scalar real comparison" begin + g = golden_val("x"; value_real=1.0, rtol=1e-6) + passed, dev, detail = compare_to_golden(qtuple(value_real=1.0 + 5e-7), g) + @test passed + @test dev ≈ 5e-7 rtol = 1e-3 + @test isempty(detail) + + passed, dev, detail = compare_to_golden(qtuple(value_real=1.0 + 1e-5), g) + @test !passed + @test dev ≈ 1e-5 rtol = 1e-3 + @test !isempty(detail) + + # Zero golden value: rtol contributes nothing, deviation is reported absolute. + g0 = golden_val("x"; value_real=0.0, rtol=1e-6, atol=0.0) + passed, dev, _ = compare_to_golden(qtuple(value_real=1e-12), g0) + @test !passed + @test dev ≈ 1e-12 + g0a = golden_val("x"; value_real=0.0, rtol=1e-6, atol=1e-9) + @test compare_to_golden(qtuple(value_real=1e-12), g0a)[1] + + # Non-finite tolerances mean "recorded, never judged": must pass even on a zero golden + # value, where atol + rtol*|gold| would otherwise evaluate to Inf + NaN. + gd = golden_val("x"; value_real=0.0, rtol=Inf, atol=Inf, class="diagnostic") + @test compare_to_golden(qtuple(value_real=42.0), gd)[1] + + @test !compare_to_golden(qtuple(value_real=nothing), g)[1] + end + + @testset "integer and token comparison" begin + # Topological integers gate on exact equality no matter how loose the rtol. + gt = golden_val("n"; value_type="integer", value_int=5, rtol=10.0, class="topological") + @test compare_to_golden(qtuple(value_int=5, value_type="integer"), gt)[1] + @test !compare_to_golden(qtuple(value_int=6, value_type="integer"), gt)[1] + + # Non-topological integers use the tolerance like a real. + gi = golden_val("n"; value_type="integer", value_int=100, rtol=0.05, class="physics_converged") + @test compare_to_golden(qtuple(value_int=103, value_type="integer"), gi)[1] + @test !compare_to_golden(qtuple(value_int=110, value_type="integer"), gi)[1] + + # Tokens are discrete: deviation 0.0 on match, 1.0 (not NaN) on mismatch so it sorts as + # a real failure. + gk = golden_val("integ"; value_type="token", value_text="riccati", class="topological") + passed, dev, _ = compare_to_golden(qtuple(value_text="riccati", value_type="token"), gk) + @test passed && dev == 0.0 + passed, dev, detail = compare_to_golden(qtuple(value_text="galerkin", value_type="token"), gk) + @test !passed && dev == 1.0 + @test occursin("riccati", detail) && occursin("galerkin", detail) + end + + @testset "array worst-element path" begin + g = golden_val("v"; value_type="json_array", value_text="[1.0,2.0,4.0]", rtol=1e-6) + + passed, dev, detail = compare_to_golden(qtuple(value_text="[1.0000000001,2.0,4.0]", value_type="json_array"), g) + @test passed + @test dev ≈ 1e-10 rtol = 1e-3 + @test isempty(detail) + + # One element out of tolerance fails the whole array and the detail names it. + passed, dev, detail = compare_to_golden(qtuple(value_text="[1.0,2.002,4.0]", value_type="json_array"), g) + @test !passed + @test dev ≈ 1e-3 rtol = 1e-3 + @test occursin("worst element 2", detail) + + # The reported deviation is the worst RELATIVE element, not the first failing one. + passed, dev, detail = compare_to_golden(qtuple(value_text="[1.001,2.0,4.02]", value_type="json_array"), g) + @test !passed + @test dev ≈ 5e-3 rtol = 1e-3 + @test occursin("worst element 3", detail) + + # A zero golden element is judged absolutely; atol=0 makes any movement on it fail. + gz = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=1e-6, atol=0.0) + @test !compare_to_golden(qtuple(value_text="[1.0e-9,1.0]", value_type="json_array"), gz)[1] + gza = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=1e-6, atol=1e-8) + @test compare_to_golden(qtuple(value_text="[1.0e-9,1.0]", value_type="json_array"), gza)[1] + + # Non-finite tolerances never judge, including zero golden elements (the Inf*0 = NaN guard). + gdz = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=Inf, atol=Inf, class="diagnostic") + @test compare_to_golden(qtuple(value_text="[5.0,99.0]", value_type="json_array"), gdz)[1] + + # Complex encoding: [re, im] pairs go through the same worst-element machinery. + gc = golden_val("v"; value_type="json_array", value_text="[[1.0,0.5],[2.0,-1.0]]", rtol=1e-6) + @test compare_to_golden(qtuple(value_text="[[1.0,0.5],[2.0,-1.0]]", value_type="json_array"), gc)[1] + passed, _, detail = compare_to_golden(qtuple(value_text="[[1.0,0.5],[2.0,-1.01]]", value_type="json_array"), gc) + @test !passed + @test occursin("worst element 2", detail) + + passed, _, detail = compare_to_golden(qtuple(value_text="[1.0,2.0]", value_type="json_array"), g) + @test !passed + @test occursin("length", detail) + end + + @testset "type-change handling" begin + g = golden_val("x"; value_real=1.0) + passed, dev, detail = compare_to_golden(qtuple(value_text="[1.0]", value_type="json_array"), g) + @test !passed && isnan(dev) + @test occursin("type changed", detail) + + # A value_type change through build_golden_values resets class and tolerances to + # provisional: measurements made under the old type must not carry over. + spec = qspec("x") + prior = Dict("x" => golden_val("x"; value_type="integer", value_int=5, rtol=0.0, + class="topological", basis="measured", drift=1e-8, spread=1e-7, at="mpsi=1024")) + vals = @test_logs (:warn, r"value_type changed") build_golden_values( + [extracted_q("x"; value_real=5.0)], [spec], prior) + @test vals["x"].class == "physics_converged" + @test vals["x"].rtol == CLASS_DEFAULT_TOLERANCE["physics_converged"].rtol + @test vals["x"].tolerance_basis == "class-default (provisional)" + @test isnan(vals["x"].plateau_drift) && isnan(vals["x"].platform_spread) + + # Same type carries the prior's hard-won measurements forward unchanged. + prior2 = Dict("x" => golden_val("x"; value_real=1.0, rtol=3e-7, atol=1e-12, + basis="measured", drift=1e-8, spread=2e-7, at="mpsi=1024")) + vals2 = build_golden_values([extracted_q("x"; value_real=1.5)], [spec], prior2) + @test vals2["x"].value_real == 1.5 + @test vals2["x"].rtol == 3e-7 && vals2["x"].atol == 1e-12 + @test vals2["x"].tolerance_basis == "measured" + @test vals2["x"].plateau_drift == 1e-8 && vals2["x"].platform_spread == 2e-7 + + # Missing extractions, checksums, and quantities without a spec never become entries. + vals3 = build_golden_values( + [extracted_q("gone"; value_type="missing"), + extracted_q("x"; value_text="abc123", value_type="checksum"), + extracted_q("unspecced"; value_real=1.0)], + [spec], nothing) + @test isempty(vals3) + end + + @testset "load-validation refusals" begin + write(golden_path("bad_class"), """ + [values.x] + class = "definitely_not_a_class" + rtol = 1.0e-6 + value = 1.0 + """) + @test_throws ErrorException load_golden("bad_class") + + # A gating entry with no finite rtol is malformed whether it was hand-edited or merged. + write(golden_path("no_rtol"), """ + [values.x] + class = "physics_converged" + value = 1.0 + """) + @test_throws ErrorException load_golden("no_rtol") + + # An rtol below the recorded platform spread is a gate no second platform can pass; + # enforced on every load so a hand edit cannot ship what save_golden refused to write. + write(golden_path("too_tight"), """ + [values.x] + class = "physics_converged" + rtol = 1.0e-9 + platform_spread = 1.0e-6 + value = 1.0 + """) + @test_throws ErrorException load_golden("too_tight") + + # Non-gating classes are exempt: they are recorded, never judged. + write(golden_path("diag_inf"), """ + [values.rt] + class = "diagnostic" + rtol = inf + atol = inf + value = 12.5 + """) + loaded = load_golden("diag_inf") + @test loaded !== nothing && !is_gating(loaded.values["rt"]) + + @test load_golden("no_such_case") === nothing + @test !has_golden("no_such_case") + end + + @testset "save_golden refusals and round trip" begin + meta = GoldenMeta("rt_case", 1, "2026-08-31", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4) + + @test_throws ErrorException save_golden(meta, Dict("empty" => golden_val("empty"))) + @test_throws ErrorException save_golden(meta, + Dict("tight" => golden_val("tight"; value_real=1.0, rtol=1e-9, spread=1e-6))) + + vals = Dict( + "a" => golden_val("a"; value_real=2.5, rtol=1e-6, drift=1e-8, spread=1e-7, at="mpsi=512"), + "n" => golden_val("n"; value_type="integer", value_int=7, rtol=0.0, class="topological"), + "v" => golden_val("v"; value_type="json_array", value_text="[1.0,2.0]", rtol=1e-6)) + save_golden(meta, vals) + back = load_golden("rt_case") + @test back.meta.commit == "deadbeef" && back.meta.golden_version == 1 + @test back.values["a"].value_real == 2.5 + @test back.values["a"].rtol == 1e-6 + @test back.values["a"].plateau_drift == 1e-8 && back.values["a"].platform_spread == 1e-7 + @test back.values["n"].value_int == 7 && back.values["n"].class == "topological" + @test back.values["v"].value_text == "[1.0,2.0]" + end + + @testset "infer_class" begin + @test infer_class(qspec("runtime"; type="runtime")) == "diagnostic" + @test infer_class(qspec("nstep"; type="int_scalar")) == "diagnostic" + @test infer_class(qspec("msing"; type="int_scalar")) == "topological" + @test infer_class(qspec("integrator"; type="token", extract="toml_key:ForceFreeStates.integrator")) == "topological" + @test infer_class(qspec("q0")) == "equilibrium_scalar" + @test infer_class(qspec("et1_re")) == "physics_converged" + end + + @testset "golden check counting and crash classification" begin + db = open_database(joinpath(mktempdir(), "check.sqlite")) + case = CaseSpec("count_case", "synthetic", "", [ + qspec("a"), + qspec("rt"; type="runtime", extract=""), + qspec("c"; type="int_scalar"), + qspec("d"), + qspec("e"), + qspec("f")], "example", Dict{String,Any}()) + + meta = GoldenMeta("count_case", 1, "2026-08-31", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4) + save_golden(meta, Dict( + "a" => golden_val("a"; value_real=1.0, rtol=1e-6), + "c" => golden_val("c"; value_type="integer", value_int=5, rtol=0.0, class="topological"), + "d" => golden_val("d"; value_real=3.0, rtol=Inf, atol=Inf, class="diagnostic"), + "f" => golden_val("f"; value_real=9.0, rtol=1e-6), + "orphan" => golden_val("orphan"; value_real=7.0, rtol=1e-6))) + + # a passes, c fails (5 → 6), d moves wildly but is diagnostic, e has no golden, + # f is golden-pinned but missing from the run, orphan has no spec left in the case. + store_run(db, "hash1", "hash1", "", "", "count_case", 1.0, [ + extracted_q("a"; value_real=1.0 + 1e-8), + extracted_q("rt"; value_real=33.0), + extracted_q("c"; value_int=6, value_type="integer"), + extracted_q("d"; value_real=300.0), + extracted_q("e"; value_real=1.0)]) + s = report_golden_check(db, case, "hash1") + @test s.n_pass == 1 + @test s.n_fail == 3 # c mismatch + f missing from run + orphaned golden entry + @test s.n_informational == 1 + @test s.n_untracked == 1 # e only: an unpinned runtime is structurally un-goldenable + @test s.n_run_failed == 0 + + # A crashed run is a crash, not a tolerance failure: n_run_failed, never n_fail. + store_failed_run(db, "hash2", "hash2", "", "", "count_case", "boom: solver exploded") + s = report_golden_check(db, case, "hash2") + @test s.n_run_failed == 1 + @test s.n_fail == 0 && s.n_pass == 0 + + # No golden file: every count zero, so the caller's zero-coverage guard (a --check that + # gated nothing must exit red) can see that nothing was checked. + nocase = CaseSpec("never_pinned", "synthetic", "", [qspec("a")], "example", Dict{String,Any}()) + s = report_golden_check(db, nocase, "hash1") + @test s == (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=0) + close_database(db) + end +end From f00317e5369bf217addc2e7eebcb236b644d0dbd Mon Sep 17 00:00:00 2001 From: d-burg Date: Wed, 23 Sep 2026 12:34:49 -0400 Subject: [PATCH 16/30] Regression - MINOR - Trim the golden-harness case and template comments Co-Authored-By: Claude Opus 5.5 --- regression-harness/src/runner.jl | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/regression-harness/src/runner.jl b/regression-harness/src/runner.jl index f59ddec60..ad46c0931 100644 --- a/regression-harness/src/runner.jl +++ b/regression-harness/src/runner.jl @@ -124,13 +124,8 @@ end """ -# Fixed-Q probe of the SLAYER inner-layer dispersion Δ(Q) on the DIII-D-like 2/1 surface. -# The adaptive AMR scan samples in gpec.h5 cannot be pinned (sample locations move under any -# refinement change), so this evaluates Δ(Q) on a fixed 4×4 grid over Re(Q), Im(Q) ∈ [-10, 10] -# instead — a refinement-stable pin of the dispersion curve itself. The layer parameters are the -# DIII-D-like SLAYER deck's own 2/1 surface values (Tearing/PerSurface), quoted so the case is -# self-contained and probes the SOLVER alone (bt is a placeholder — the dispersion solve never -# reads it); the parameter chain producing these numbers is pinned separately by diiid_slayer_n1. +# SLAYER dispersion Δ(Q) on a fixed 4×4 Q grid at the DIII-D-like 2/1 surface's layer parameters, +# a refinement-stable pin of the solver alone (AMR sample locations cannot be pinned; bt is unread). const COMPUTED_SLAYER_DELTA_PROBE_SCRIPT_TEMPLATE = """ using Pkg %INSTANTIATE% @@ -157,13 +152,9 @@ h5open(ARGS[1], "w") do fid end %RUNINFO% """ -# External-reference validation: GPEC's del_s Riccati solver against Fitzpatrick, "Tearing Mode -# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3 — see the case TOML header for -# the validation evidence. Prescribing the normalized parameters (D_norm = 1, -# P_perp = P_tor = Phat, Q_e = Qhat/(1+1/tau)) makes the solver's internal Q_hat equal the -# book's Qhat_*, so the grid below is exactly the figures' axes. tau = 1 is an explicit pinned -# assumption (not stated in the captions). The grid starts just inside Phat = 0, which is a -# singular edge of the model (alpha vanishes and the large-q boundary form degenerates). +# del_s over Fitzpatrick (IOP 2023) figs 6.2/6.3's (Qhat, Phat) axes: D_norm = 1, P_perp = P_tor = Phat +# and Q_e = Qhat/(1+1/tau) make the solver's Q_hat the book's. tau = 1 is assumed; Phat starts just +# inside 0, a singular edge of the model. const COMPUTED_SLAYER_DELS_FITZPATRICK_SCRIPT_TEMPLATE = """ using Pkg %INSTANTIATE% From 67a2c40cb8f23df160c8353ecc7f4cc63e8eebb2 Mon Sep 17 00:00:00 2001 From: d-burg Date: Wed, 23 Sep 2026 12:34:49 -0400 Subject: [PATCH 17/30] Regression - BUGFIX - Close the golden gates that a hand edit or a re-pin could open - Validate every entry on load and save: gating classes need a finite, non-negative rtol and atol (an atol of inf passed every comparison), rtol may not undercut a recorded plateau_drift or platform_spread, and a "measured" basis must record one of them. - Judge by class, not by tolerance finiteness, so only diagnostic/unconverged go unjudged. - A quantity's class comes from its case file (new optional `class` key, else inference); a golden file whose class disagrees fails the check, so demoting a gate is a case-file change. Declare ntv_psi_nsteps diagnostic and the Fitzpatrick scan axes topological; psio infers equilibrium_scalar. - The zero-coverage guard counts gated entries, not golden files. - --update-golden requires --cases, refuses uncommitted tracked changes outside golden/, records the pinned ref's commit, skips runtimes, and prints integer/token/array moves with a flag on any move the old tolerance would have failed. - Array comparison fails a nested row that changed length and points the detail at a NaN element. - Say plainly that tolerances are provisional until measured, and that the Fitzpatrick golden pins GPEC's validated output rather than the contour-band agreement itself. Co-Authored-By: Claude Opus 5.5 --- regression-harness/cases/diiid_slayer_n1.toml | 2 +- .../cases/slayer_dels_fitzpatrick.toml | 16 +- .../cases/solovev_kinetic_ntv.toml | 1 + regression-harness/regress.jl | 20 +- regression-harness/src/config.jl | 3 +- regression-harness/src/extractor.jl | 10 +- regression-harness/src/golden.jl | 259 ++++++++++------ regression-harness/src/types.jl | 9 +- regression-harness/test/runtests.jl | 287 +++++++++++++----- 9 files changed, 410 insertions(+), 197 deletions(-) diff --git a/regression-harness/cases/diiid_slayer_n1.toml b/regression-harness/cases/diiid_slayer_n1.toml index 6a1a4edf5..f969fe8cf 100644 --- a/regression-harness/cases/diiid_slayer_n1.toml +++ b/regression-harness/cases/diiid_slayer_n1.toml @@ -10,7 +10,6 @@ name = "diiid_slayer_n1" description = "DIII-D-like H-mode equilibrium, n=1, SLAYER tearing-mode analysis (uncoupled per-surface, AMR, validity-gated)" example_dir = "examples/DIIID-like_SLAYER_example" -# Per-surface SLAYER layer parameters (geometry + dimensionless) # Which formalism produced the Δ′ this case's layer physics consumes. The Δ′ matrix lands on # one canonical SingularSurfaces/ path whichever integrator computed it, so pinning the deck's # declared choice is what keeps a BVP-derived gold from being compared against a Galerkin run. @@ -22,6 +21,7 @@ label = "integrator (Δ′ provenance)" noise_threshold = 0 order = 1 +# Per-surface SLAYER layer parameters (geometry + dimensionless) [quantities.slayer_ising] h5path = "Tearing/PerSurface/rational_index" type = "real_vector" diff --git a/regression-harness/cases/slayer_dels_fitzpatrick.toml b/regression-harness/cases/slayer_dels_fitzpatrick.toml index cc3c226bf..7e39e48c1 100644 --- a/regression-harness/cases/slayer_dels_fitzpatrick.toml +++ b/regression-harness/cases/slayer_dels_fitzpatrick.toml @@ -1,11 +1,12 @@ # Regression case: GPEC's del_s Riccati layer-width solver against Fitzpatrick, "Tearing Mode # Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3 (p. 6-10). # -# This is the suite's only case pinned to an EXTERNAL published reference rather than to GPEC's own -# previous output: a failure means we no longer reproduce the textbook. Verified at eight points -# against the published contour bands, e.g. (Q,P) = (4.0, 0.5) -> Re 2.008 against the 1.9-2.1 -# contours, (0.5, 4.0) -> Re 1.537 against the 1.5 contour. The magnitudes also match the book's -# table 6.2 reactor estimates (|delta_s| 2.95-4.69 mm) at DIII-D-like parameters. +# Verified by hand at eight points against the book's published contour bands, e.g. (Q,P) = +# (4.0, 0.5) -> Re 2.008 against the 1.9-2.1 contours, (0.5, 4.0) -> Re 1.537 against the 1.5 +# contour; the magnitudes also match its table 6.2 reactor estimates (|delta_s| 2.95-4.69 mm) at +# DIII-D-like parameters. The golden file pins GPEC's output at that validated state to a +# provisional class-default tolerance: a failure means the validated numbers moved, and any re-pin +# must be re-checked against the figures, since the gate itself is not the contour-band agreement. # # Pure ODE: no equilibrium, no geqdsk, no kinetic file. Sub-second, so it is fast-tier eligible, and # it carries no equilibrium dependence -- its golden values survive equilibrium-side re-baselining. @@ -16,7 +17,7 @@ name = "slayer_dels_fitzpatrick" description = "del_s Riccati layer width vs Fitzpatrick 2023 figs 6.2/6.3 over (Q_hat, P_hat)" kind = "computed" -# The scan axes, pinned so a silently reshaped grid is a failure rather than a re-gold. +# The scan axes, pinned exactly so a silently reshaped grid is a failure rather than a re-gold. [quantities.q_hat] h5path = "fitzpatrick/Q_hat" type = "real_vector" @@ -24,6 +25,7 @@ extract = "all_real" label = "grid Q_hat (= Q_*/D^4)" noise_threshold = 0 order = 10 +class = "topological" # exact input, not a converged output [quantities.p_hat] h5path = "fitzpatrick/P_hat" @@ -32,6 +34,7 @@ extract = "all_real" label = "grid P_hat (= P/D^6)" noise_threshold = 0 order = 11 +class = "topological" # exact input, not a converged output # tau is not stated in the book's figure captions; tau = 1 (table 5.1, low-field reactor) is the # assumption this reproduction rests on, so it is pinned rather than left implicit. @@ -42,6 +45,7 @@ extract = "value" label = "tau (assumed; not given in the figure captions)" noise_threshold = 0 order = 12 +class = "topological" # exact input, not a converged output [quantities.dels_db_re] h5path = "fitzpatrick/dels_db_re" diff --git a/regression-harness/cases/solovev_kinetic_ntv.toml b/regression-harness/cases/solovev_kinetic_ntv.toml index a9a4d5324..87f1d9667 100644 --- a/regression-harness/cases/solovev_kinetic_ntv.toml +++ b/regression-harness/cases/solovev_kinetic_ntv.toml @@ -26,6 +26,7 @@ extract = "value" label = "NTV ψ quadrature evaluations" noise_threshold = 0 order = 11 +class = "diagnostic" # adaptive step count: describes the numerics, never gates # Stability anchors — confirm the upstream FFS/PE stages feeding the NTV diagnostic. [quantities.et_real] diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 7b823ed86..356ddfeb5 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -133,7 +133,8 @@ Options: run always exits non-zero regardless) --check Run the working tree and compare against the committed golden values (regression-harness/golden/.toml). This is the mode CI gates on. - --update-golden Regenerate the golden values from a fresh run. Requires --reason. + --update-golden Regenerate the golden values from a fresh run. Requires --cases and + --reason, and refuses a working tree with uncommitted changes. --reason "..." Why the goldens changed. Mandatory with --update-golden: a golden change is a claim about physics that a reviewer has to evaluate. --help Print this help message @@ -144,9 +145,11 @@ Environment: Golden values: A golden file pins each quantity together with the tolerance it must be reproduced within - and the evidence behind that tolerance. Tolerances are derived from measured convergence - plateau drift and cross-platform spread — never widened to make a check pass. A failing - --check is fixed by explaining the physics or fixing the regression, not by loosening rtol. + and the evidence behind that tolerance. Tolerances must come from measured convergence + plateau drift and cross-platform spread; until measured they are provisional class defaults + and say so. A quantity's class is declared in its case file (`class = ...`) or inferred, + never set in the golden file. A failing --check is fixed by explaining the physics or + fixing the regression, not by loosening rtol. Exit status: 0 all runs completed (and, with --fail-on-change, nothing changed) @@ -229,6 +232,9 @@ function main(args=ARGS) error("--update-golden requires --reason \"...\": a golden change is a claim about physics, " * "and the reviewer needs to know what changed and why") end + if opts.update_golden && isempty(opts.cases) + error("--update-golden requires --cases: goldens are regenerated one deliberate case at a time") + end # Resolve refs. Golden modes judge the working tree, so they default to it rather than # requiring the caller to spell out --refs local. @@ -296,7 +302,7 @@ function main(args=ARGS) n_golden_fail += summary.n_fail n_golden_crashed += summary.n_run_failed n_untracked += summary.n_untracked - has_golden(case_spec.name) ? (n_checked += 1) : (n_no_golden += 1) + summary.n_pass + summary.n_fail > 0 ? (n_checked += 1) : (n_no_golden += 1) else summary = if length(resolved_refs) == 2 report_two_ref_comparison(db, case_spec, @@ -324,11 +330,11 @@ function main(args=ARGS) if opts.check && n_checked == 0 # A green gate that checked nothing is worse than a red one: a deleted or # typo-named golden file must not be indistinguishable from a passing check. - @error "--check ran $n_no_golden case(s) but found no golden file for any of them — nothing was actually gated" + @error "--check ran $n_no_golden case(s) but none has a gating golden entry — nothing was actually gated" exit(1) end if opts.check && n_no_golden > 0 - @warn "$n_no_golden of $(n_no_golden + n_checked) requested case(s) have no golden file and were not gated" + @warn "$n_no_golden of $(n_no_golden + n_checked) requested case(s) have no gating golden entry and were not gated" end if n_untracked > 0 @info "$n_untracked tracked quantity/quantities have no golden value yet (not gating)" diff --git a/regression-harness/src/config.jl b/regression-harness/src/config.jl index 4bee9d408..c1a50a4a2 100644 --- a/regression-harness/src/config.jl +++ b/regression-harness/src/config.jl @@ -23,7 +23,8 @@ function load_case(filepath::String)::CaseSpec qty_data["extract"], get(qty_data, "label", qty_name), get(qty_data, "noise_threshold", 1e-10), - get(qty_data, "order", 1000) + get(qty_data, "order", 1000), + get(qty_data, "class", "") ) ) end diff --git a/regression-harness/src/extractor.jl b/regression-harness/src/extractor.jl index 2d1d632f2..a4747158c 100644 --- a/regression-harness/src/extractor.jl +++ b/regression-harness/src/extractor.jl @@ -113,13 +113,8 @@ function apply_extraction(spec::QuantitySpec, raw)::ExtractedQuantity return ExtractedQuantity(name, label, nothing, nothing, json_str, "json_array", threshold) elseif startswith(spec.extract, "toml_key:") - # "toml_key:": pin a deck's declared control value, read out of the - # `Input/gpec_toml_raw` rerun snapshot. Used to record which formalism a case asked - # for, because the Δ′ matrix is written to one canonical `SingularSurfaces/` path - # whichever integrator produced it, so the numbers alone no longer say. Reading it - # from `Input/` respects the schema rule that control values live nowhere else. - # A key the deck leaves unset pins as "", so a deck that starts or stops - # declaring one is itself a change rather than being silently absorbed. + # Pin a deck's declared control value from the Input/ snapshot; an unset key pins as + # "" so starting or stopping to declare it is itself a change. keypath = spec.extract[(length("toml_key:")+1):end] table = TOML.parse(raw isa AbstractString ? raw : String(raw)) node = table @@ -150,6 +145,7 @@ function _json_element_diff(a, b)::Float64 (isnan(Float64(a)) && isnan(Float64(b))) && return 0.0 return abs(Float64(a) - Float64(b)) elseif a isa Vector && b isa Vector + length(a) == length(b) || return Inf return sqrt(sum(_json_element_diff(ai, bi)^2 for (ai, bi) in zip(a, b))) else return Inf diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 94170f1d8..5dad40149 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -8,16 +8,18 @@ of every quantity plus the tolerance it must be reproduced within and the eviden tolerance. The tolerance policy is the load-bearing part. A tolerance is a claim about how much a quantity -may legitimately move, so it is *derived* — from the residual drift at a convergence plateau and -from the spread measured across platforms — never chosen to make a check pass. `update_golden` -refuses to write a tolerance below the recorded platform spread, and a failing check is fixed by -explaining the physics or fixing the regression, not by widening the bound. +may legitimately move, so it must come from measurement — the residual drift at a convergence +plateau and the spread across platforms — never be chosen to make a check pass. Until measured, an +entry carries a provisional class default and is stamped as such. `save_golden` and `load_golden` +refuse a tolerance below either recorded measurement, and a failing check is fixed by explaining +the physics or fixing the regression, not by widening the bound. """ """ How strictly a quantity must reproduce, and why. - - `topological` — integer counts and mode numbers; exact equality + - `topological` — integer counts, mode numbers, and exact inputs such as echoed scan grids; + exact equality - `equilibrium_scalar` — spline/quadrature outputs with no adaptive branching; very tight - `physics_converged` — the quantities golden values exist for (δW, Δ′, torque, growth rates); tolerance comes from the measured convergence plateau and platform spread @@ -25,10 +27,15 @@ How strictly a quantity must reproduce, and why. describe the numerics rather than the physics - `unconverged` — measured and found to have no plateau; tracked differentially, never pinned, so that a known-unconverged quantity is visibly excluded instead of quietly given a wide bound + +A quantity's class comes from its case file (an explicit `class` key, else `infer_class`), never +from the golden file, so demoting a gate to a non-gating class is a reviewed case-file change. """ const TOLERANCE_CLASSES = ("topological", "equilibrium_scalar", "physics_converged", "diagnostic", "unconverged") -"""Classes whose failure fails the run. `diagnostic` and `unconverged` are reported only.""" +""" +Classes whose failure fails the run. `diagnostic` and `unconverged` are reported only. +""" const GATING_CLASSES = ("topological", "equilibrium_scalar", "physics_converged") """ @@ -56,8 +63,8 @@ One quantity's pinned value and the terms it is judged by. - `value_real` / `value_int` / `value_text` — the pinned value in its stored form - `rtol` / `atol` — pass when `|x - gold| <= atol + rtol*|gold|` - `class` — one of `TOLERANCE_CLASSES` - - `tolerance_basis` — how the tolerance was arrived at: "measured" or - "class-default (provisional)" + - `tolerance_basis` — how the tolerance was arrived at: "measured" (requires a recorded + `plateau_drift` or `platform_spread`) or "class-default (provisional)" - `plateau_drift` — relative movement over the final refinement of the convergence scan; NaN when not measured - `platform_spread` — relative disagreement between reference platforms; NaN when not measured @@ -81,6 +88,28 @@ end is_gating(g::GoldenValue) = g.class in GATING_CLASSES +""" +Refuse an entry that would gate more quietly than its evidence allows. Run on every load as well as +every save, so a hand edit or a merge taking the wrong side is caught as surely as a bad write. +""" +function validate_golden_value(g::GoldenValue, where::AbstractString) + g.class in TOLERANCE_CLASSES || error("$where: quantity '$(g.name)' has unknown class '$(g.class)'") + if is_gating(g) + (isfinite(g.rtol) && g.rtol >= 0 && isfinite(g.atol) && g.atol >= 0) || + error("$where: gating quantity '$(g.name)' needs a finite, non-negative rtol and atol (got rtol=$(g.rtol), atol=$(g.atol))") + for (label, m) in (("platform_spread", g.platform_spread), ("plateau_drift", g.plateau_drift)) + isfinite(m) && g.rtol < m && + error( + "$where: quantity '$(g.name)' has rtol $(g.rtol) below its measured $label $m. " * + "Widen it deliberately, with the measurement recorded, or reduce the $label before pinning." + ) + end + end + g.tolerance_basis == "measured" && !isfinite(g.plateau_drift) && !isfinite(g.platform_spread) && + error("$where: quantity '$(g.name)' claims a measured tolerance but records neither plateau_drift nor platform_spread") + return nothing +end + """ Provenance for a whole golden file: what produced these numbers and why they last changed. """ @@ -123,17 +152,7 @@ function load_golden(case_name::AbstractString) ) values = Dict{String,GoldenValue}() for (name, v) in get(data, "values", Dict{String,Any}()) - class = get(v, "class", "physics_converged") - class in TOLERANCE_CLASSES || error("Golden file $path: quantity '$name' has unknown class '$class'") - # Enforced on every load, not only at write time: a hand edit or a merge taking the - # wrong side must not produce a gate quieter than the one save_golden refused to write. - if class in GATING_CLASSES - rt = Float64(get(v, "rtol", NaN)) - (isfinite(rt) && rt >= 0) || error("Golden file $path: gating quantity '$name' has no finite rtol — malformed or hand-edited entry") - sp = Float64(get(v, "platform_spread", NaN)) - isfinite(sp) && rt < sp && error("Golden file $path: quantity '$name' has rtol $rt below its recorded platform_spread $sp") - end - values[name] = GoldenValue( + g = GoldenValue( name, get(v, "value_type", "real"), haskey(v, "value") && v["value"] isa Real ? Float64(v["value"]) : nothing, @@ -141,12 +160,14 @@ function load_golden(case_name::AbstractString) haskey(v, "value_text") ? String(v["value_text"]) : nothing, Float64(get(v, "rtol", NaN)), Float64(get(v, "atol", 0.0)), - class, + get(v, "class", "physics_converged"), get(v, "tolerance_basis", "unspecified"), Float64(get(v, "plateau_drift", NaN)), Float64(get(v, "platform_spread", NaN)), get(v, "converged_at", "") ) + validate_golden_value(g, "Golden file $path") + values[name] = g end return (meta=meta, values=values) end @@ -154,23 +175,20 @@ end """ Write a golden file. -Refuses to emit a tolerance tighter than a recorded platform spread: a pin that no second -platform can reproduce is a broken gate, and silently loosening it later is exactly the failure -this whole mechanism exists to prevent. Raising the tolerance to cover a measured spread is a -deliberate act the caller performs, not something this function does behind the caller's back. +Refuses any entry `validate_golden_value` rejects, and any entry with no value. Raising a +tolerance to cover a measured spread is a deliberate act the caller performs, never something this +function does behind the caller's back. """ function save_golden(meta::GoldenMeta, values::Dict{String,GoldenValue}) for g in Base.values(values) if g.value_real === nothing && g.value_int === nothing && g.value_text === nothing - error("Quantity '$(g.name)': refusing to write a golden entry with no value (the run " * - "produced NaN or nothing) — a valueless gating entry fails forever and regenerating " * - "reproduces it. Exclude the quantity or fix the extraction.") - end - if isfinite(g.platform_spread) && isfinite(g.rtol) && g.rtol < g.platform_spread - error("Quantity '$(g.name)': rtol $(g.rtol) is tighter than the measured platform " * - "spread $(g.platform_spread). Widen it deliberately, with the measurement recorded, or " * - "reduce the spread before pinning.") + error( + "Quantity '$(g.name)': refusing to write a golden entry with no value (the run " * + "produced NaN or nothing) — a valueless gating entry fails forever and regenerating " * + "reproduces it. Exclude the quantity or fix the extraction." + ) end + validate_golden_value(g, "Refusing to write golden for $(meta.case)") end mkpath(GOLDEN_DIR) out = Dict{String,Any}( @@ -215,22 +233,25 @@ end Classify a quantity from its case spec, so a new case gets sensible defaults without every tolerance having to be written by hand. -Integer counts are topological; runtimes and step counts describe the numerics rather than the -physics and so are diagnostic; everything else is assumed to be a physics quantity that must be -converged, which is the conservative assumption — it gates. +An explicit `class` in the case file wins. Otherwise integer counts are topological; runtimes and +step counts describe the numerics rather than the physics and so are diagnostic; everything else +is assumed to be a physics quantity that must be converged, which is the conservative assumption — +it gates. """ function infer_class(spec::QuantitySpec)::String + if !isempty(spec.class) + spec.class in TOLERANCE_CLASSES || error("Quantity '$(spec.name)': unknown class '$(spec.class)' in its case file") + return spec.class + end spec.type == "runtime" && return "diagnostic" spec.name in ("nstep", "nstep_total") && return "diagnostic" spec.type == "int_scalar" && return "topological" - # A declared control token is an exact-match gate, like a count: there is no tolerance - # that means anything between "riccati" and "galerkin". + # A control token has no meaningful tolerance between "riccati" and "galerkin". startswith(spec.extract, "toml_key:") && return "topological" name = spec.name - # sing_psi / sing_q are deliberately absent: singular-surface locations come from a root - # search, not pure spline/quadrature, so they take the measured physics_converged path. + # sing_psi / sing_q are absent on purpose: they come from a root search, not pure quadrature. equilibrium_names = ("q0", "q95", "betat", "betan", "betap1", "betap2", "betap3", "betaj", - "li1", "li2", "li3", "volume", "crnt", "bt0", "bwall", "aratio", "kappa") + "li1", "li2", "li3", "volume", "crnt", "bt0", "bwall", "aratio", "kappa", "psio") name in equilibrium_names && return "equilibrium_scalar" return "physics_converged" end @@ -243,10 +264,8 @@ the golden value is zero) and `detail` is a human-readable reason on failure. Ar pass only when every element is within tolerance; the reported deviation is the worst element. """ function compare_to_golden(q::NamedTuple, g::GoldenValue) - # Non-finite tolerances mean "recorded, never judged" (diagnostic/unconverged); without this - # guard, gold == 0 turns atol + rtol*|gold| into Inf + NaN and the comparison is false. - within = (x, gold) -> !isfinite(g.atol) || !isfinite(g.rtol) || - abs(x - gold) <= g.atol + g.rtol * abs(gold) + # Non-gating classes are recorded, never judged; gating ones are validated finite on load. + within = (x, gold) -> !is_gating(g) || abs(x - gold) <= g.atol + g.rtol * abs(gold) if q.value_type != g.value_type return (false, NaN, "type changed: golden $(g.value_type), got $(q.value_type)") @@ -274,10 +293,7 @@ function compare_to_golden(q::NamedTuple, g::GoldenValue) return (false, NaN, "length $(length(gold)) → $(length(got))") elseif g.value_type == "token" - # A token names a discrete choice (which integrator produced the Δ′), so there is no - # tolerance to apply: it matches or the run is answering a different question than the - # gold does. Deviation is reported as 1.0 on mismatch rather than NaN so it sorts as a - # real failure in reports. + # Discrete choice, so no tolerance; deviation 1.0 (not NaN) on mismatch sorts as a failure. (q.value_text === nothing || g.value_text === nothing) && return (false, NaN, "missing token") matched = q.value_text == g.value_text return (matched, matched ? 0.0 : 1.0, matched ? "" : "$(g.value_text) → $(q.value_text)") @@ -286,7 +302,9 @@ function compare_to_golden(q::NamedTuple, g::GoldenValue) return (false, NaN, "unsupported value type $(g.value_type)") end -"""Worst-element comparison for array quantities, shared by the real and complex encodings.""" +""" +Worst-element comparison for array quantities, shared by the real and complex encodings. +""" function _compare_arrays(got, gold, g::GoldenValue) worst_rel = 0.0 worst_idx = 0 @@ -294,12 +312,11 @@ function _compare_arrays(got, gold, g::GoldenValue) for i in eachindex(gold) a = _json_element_abs(gold[i]) d = _json_element_diff(gold[i], got[i]) - rel = a == 0.0 ? d : d / a - # Same non-finite guard as the scalar path: Inf tolerances mean "recorded, never - # judged", and Inf*0 = NaN would otherwise mark a zero gold element as failing. - ok = !isfinite(g.atol) || !isfinite(g.rtol) || d <= g.atol + g.rtol * a + # A NaN on one side counts as the worst possible element, so the detail points at it. + rel = isnan(d) ? Inf : (a == 0.0 ? d : d / a) + ok = !is_gating(g) || d <= g.atol + g.rtol * a ok || (all_ok = false) - if rel > worst_rel + if rel > worst_rel || (worst_idx == 0 && !ok) worst_rel = rel worst_idx = i end @@ -341,8 +358,7 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St for spec in case_spec.quantities g = get(golden.values, spec.name, nothing) if g === nothing - # Runtime and checksums are structurally un-goldenable (no tolerance semantics), so - # they must not inflate the untracked count that flags genuinely unpinned physics. + # Runtimes and checksums are never pinned, so they do not count as untracked physics. (spec.type == "runtime" || spec.extract == "checksum") && continue n_untracked += 1 continue @@ -353,11 +369,17 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St n_fail += 1 continue end - # SQLite NULLs surface as `missing`, which the === nothing guards in compare_to_golden - # never match; normalize here as the update path already does. + # The golden file's class must be the one the case declares, so a demotion cannot hide there. + declared = infer_class(spec) + if g.class != declared + push!(rows, [spec.label, g.class, "CLASS", "—", "** FAIL ** case declares $declared; regenerate goldens"]) + n_fail += 1 + continue + end + # SQLite NULLs surface as `missing`; normalize them to nothing as the update path does. q = (label=q_raw.label, value_real=_column(q_raw.value_real, nothing), - value_int=_column(q_raw.value_int, nothing), value_text=_column(q_raw.value_text, nothing), - value_type=q_raw.value_type, noise_threshold=q_raw.noise_threshold) + value_int=_column(q_raw.value_int, nothing), value_text=_column(q_raw.value_text, nothing), + value_type=q_raw.value_type, noise_threshold=q_raw.noise_threshold) passed, deviation, detail = compare_to_golden(q, g) gating = is_gating(g) status = if !gating @@ -379,10 +401,7 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St push!(rows, [spec.label, g.class, dev, tol, status * (isempty(detail) ? "" : " $detail")]) end - # Golden entries with no matching case quantity fail regardless of class: a renamed or - # removed quantity would otherwise silently delete its gate (the new name enters as merely - # "untracked"), and a check that quietly stops checking something is worse than one that - # fails. The fix is to regenerate the goldens for the reshaped case, with a reason. + # An orphaned entry fails regardless of class: a renamed quantity must not silently drop its gate. spec_names = Set(spec.name for spec in case_spec.quantities) for name in sort(collect(keys(golden.values))) name in spec_names && continue @@ -424,24 +443,42 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational, n_run_failed=0) end +""" +Uncommitted changes to tracked files under `repo_root`, ignoring `exclude_dir` (the goldens themselves). +""" +function uncommitted_changes(repo_root::AbstractString, exclude_dir::AbstractString) + rel = relpath(exclude_dir, repo_root) + return String(strip(read(`git -C $repo_root status --porcelain --untracked-files=no -- . ":(exclude)$rel"`, String))) +end + """ Regenerate a case's golden file from a completed run, reporting what moved. -Prints an old→new delta for every quantity whose value changed, so the diff a reviewer sees in -git is accompanied by the size of each move. +Prints an old→new delta for every quantity whose value or class changed, flagging any move that +the old tolerance would have failed, so the diff a reviewer sees in git comes with the size and +significance of each move. Refuses a working tree with uncommitted changes outside the golden +directory, because the recorded commit is what a reviewer uses to reproduce a disputed number. """ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String, - reason::String, repo_root::String) + reason::String, repo_root::String) info = get_run_info(db, commit_hash, case_spec.name) (info === nothing || !info.success) && error("Cannot update goldens for '$(case_spec.name)': the run did not succeed") + if commit_hash == LOCAL_REF + dirty = uncommitted_changes(repo_root, GOLDEN_DIR) + isempty(dirty) || error( + "Cannot update goldens for '$(case_spec.name)': the working tree has uncommitted " * + "changes, so the recorded commit would not reproduce these values. Commit first.\n$dirty" + ) + end - # SQLite NULLs come back as `missing`, which the Union{...,Nothing} fields reject; `_column` - # normalizes both absent forms to nothing. extracted = ExtractedQuantity[] for (name, q) in get_quantities(db, commit_hash, case_spec.name) - push!(extracted, ExtractedQuantity(name, String(_column(q.label, name)), - _column(q.value_real, nothing), _column(q.value_int, nothing), - _column(q.value_text, nothing), q.value_type, q.noise_threshold)) + push!( + extracted, + ExtractedQuantity(name, String(_column(q.label, name)), + _column(q.value_real, nothing), _column(q.value_int, nothing), + _column(q.value_text, nothing), q.value_type, q.noise_threshold) + ) end previous = load_golden(case_spec.name) @@ -452,71 +489,93 @@ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash: meta = GoldenMeta(case_spec.name, previous === nothing ? 1 : previous.meta.golden_version + 1, Dates.format(Dates.now(), "yyyy-mm-dd"), - strip(read(`git -C $repo_root rev-parse --short HEAD`, String)), + strip(read(`git -C $repo_root rev-parse --short $(commit_hash == LOCAL_REF ? "HEAD" : commit_hash)`, String)), reason, fp.julia_version, fp.os_arch, fp.manifest_sha, fp.nthreads, fp.blas_threads) - if commit_hash == LOCAL_REF && !isempty(strip(read(`git -C $repo_root status --porcelain`, String))) - # The numbers came from uncommitted source; a clean HEAD checkout will not reproduce - # them, and the commit field is the one a reviewer uses to reproduce a disputed number. - meta = GoldenMeta(meta.case, meta.golden_version, meta.generated_at, - meta.commit * "-dirty", meta.reason, meta.julia_version, meta.os_arch, - meta.manifest_sha, meta.nthreads, meta.blas_threads) - @warn "Working tree is dirty: golden provenance recorded as $(meta.commit). Commit first if these values are meant to be reproducible." - end println() println("Golden update: $(case_spec.name) (v$(previous === nothing ? 0 : previous.meta.golden_version) → v$(meta.golden_version))") if existing !== nothing for name in sort(collect(keys(existing))) if !haskey(values, name) - println(@sprintf(" %-34s REMOVED — extraction returned missing (renamed h5 path?) or the quantity left the case. This deletes its gate; confirm it is intentional.", name)) + println( + @sprintf( + " %-34s REMOVED — extraction returned missing (renamed h5 path?) or the quantity left the case. This deletes its gate; confirm it is intentional.", + name + ) + ) end end for (name, g) in sort(collect(values); by=first) old = get(existing, name, nothing) old === nothing && (println(@sprintf(" %-34s NEW", name)); continue) - if g.value_real !== nothing && old.value_real !== nothing && g.value_real != old.value_real - rel = old.value_real == 0 ? Inf : abs(g.value_real - old.value_real) / abs(old.value_real) - println(@sprintf(" %-34s %.9g → %.9g (rel %.2e)", name, old.value_real, g.value_real, rel)) - elseif g.value_text !== nothing && old.value_text !== nothing && g.value_text != old.value_text - println(@sprintf(" %-34s array changed", name)) - end + old.class == g.class || + println(@sprintf(" %-34s class %s → %s (tolerances reset to provisional)", name, old.class, g.class)) + line = describe_golden_change(old, g) + line === nothing || println(@sprintf(" %-34s %s", name, line)) end end path = save_golden(meta, values) println("Wrote $path ($(length(values)) quantities)") provisional = count(g -> startswith(g.tolerance_basis, "class-default"), Base.values(values)) - provisional > 0 && println(" $provisional quantity/quantities still carry provisional class-default tolerances — " * - "replace them with measured plateau drift and platform spread before relying on this as a gate.") + provisional > 0 && println( + " $provisional quantity/quantities still carry provisional class-default tolerances — " * + "replace them with measured plateau drift and platform spread before relying on this as a gate." + ) println() return path end +""" +Describe how a regenerated entry's value moved from the old one, or `nothing` if it did not. + +The deviation is the one `compare_to_golden` computes against the old entry, and a move the old +tolerance would have failed is flagged so it cannot pass unnoticed as a routine re-pin. +""" +function describe_golden_change(old::GoldenValue, new::GoldenValue) + old.value_type == new.value_type || return "type $(old.value_type) → $(new.value_type)" + (old.value_real, old.value_int, old.value_text) == (new.value_real, new.value_int, new.value_text) && return nothing + q = (value_real=new.value_real, value_int=new.value_int, value_text=new.value_text, value_type=new.value_type) + passed, dev, _ = compare_to_golden(q, old) + what = if new.value_type == "real" + @sprintf("%.9g → %.9g", something(old.value_real, NaN), something(new.value_real, NaN)) + elseif new.value_type == "integer" + "$(old.value_int) → $(new.value_int)" + elseif new.value_type == "token" + "$(old.value_text) → $(new.value_text)" + else + "array[$(length(JSON.parse(new.value_text; allownan=true)))] changed" + end + rel = isnan(dev) ? "—" : @sprintf("%.2e", dev) + flag = is_gating(old) && !passed ? " ** EXCEEDS the old tolerance: a regression unless --reason explains it **" : "" + return "$what (rel $rel)$flag" +end + """ Build golden entries from a set of freshly extracted quantities. -`existing` carries forward the tolerance, class, and evidence already recorded for a quantity, so +`existing` carries forward the tolerance and evidence already recorded for a quantity, so regenerating values after a physics change does not silently reset hard-won measurements to -provisional defaults. +provisional defaults. A change of value type or of the case-declared class invalidates them. """ function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector{QuantitySpec}, - existing::Union{Dict{String,GoldenValue},Nothing}) + existing::Union{Dict{String,GoldenValue},Nothing}) spec_by_name = Dict(s.name => s for s in specs) values = Dict{String,GoldenValue}() for eq in extracted eq.value_type == "missing" && continue spec = get(spec_by_name, eq.name, nothing) spec === nothing && continue + # Checksums have no notion of "close" and runtimes are wall-clock, so neither is pinned. + (eq.value_type == "checksum" || spec.type == "runtime") && continue + class = infer_class(spec) prior = existing === nothing ? nothing : get(existing, eq.name, nothing) if prior !== nothing && prior.value_type != eq.value_type - # A type change invalidates the class and every measurement made under the old type; - # carrying a topological rtol=0 onto a float (or a float rtol onto a count) mis-gates. - @warn "Golden '$(eq.name)': value_type changed $(prior.value_type) → $(eq.value_type); resetting class and tolerances to provisional" + @warn "Golden '$(eq.name)': value_type changed $(prior.value_type) → $(eq.value_type); resetting tolerances to provisional" + prior = nothing + elseif prior !== nothing && prior.class != class + @warn "Golden '$(eq.name)': class changed $(prior.class) → $class; resetting tolerances to provisional" prior = nothing end - class = prior === nothing ? infer_class(spec) : prior.class - # Checksums have no notion of "close", so they cannot carry a tolerance; they stay a - # same-machine differential tool rather than a golden gate. - eq.value_type == "checksum" && continue if prior === nothing defaults = CLASS_DEFAULT_TOLERANCE[class] rtol, atol, basis = defaults.rtol, defaults.atol, "class-default (provisional)" diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl index 4c7393b06..08bc9db99 100644 --- a/regression-harness/src/types.jl +++ b/regression-harness/src/types.jl @@ -4,6 +4,9 @@ Shared data structures for the regression harness. """ Specification for a single quantity to extract from gpec.h5. + +`class` is the golden tolerance class declared in the case file, or empty to let `infer_class` +choose; declaring it is the only way to make a quantity non-gating that inference would gate. """ struct QuantitySpec name::String @@ -14,16 +17,18 @@ struct QuantitySpec label::String # Human-readable label for reports noise_threshold::Float64 # Absolute changes below this are noise order::Int # Display order in reports (lower = earlier) + class::String # Declared golden tolerance class; empty = inferred end """ Specification for a test case: what to run and what to extract. `kind` selects the runner backend: + - "gpec_run" (default) — run GPEC end-to-end on `example_dir`, extract from `gpec.h5` - "computed" — run a self-contained Julia computation that writes a small h5 - (no `example_dir` required); used for analytic/reference cases - like the GGJ inner-layer benchmark. + (no `example_dir` required); used for analytic/reference cases + like the GGJ inner-layer benchmark. """ struct CaseSpec name::String diff --git a/regression-harness/test/runtests.jl b/regression-harness/test/runtests.jl index 86cde0a18..49666b601 100644 --- a/regression-harness/test/runtests.jl +++ b/regression-harness/test/runtests.jl @@ -18,11 +18,11 @@ for f in ("types.jl", "env.jl", "config.jl", "database.jl", "utils.jl", "extract include(joinpath(HARNESS_DIR, "src", f)) end -qspec(name; type="real_scalar", extract="value", label=name, noise=0.0, order=1) = - QuantitySpec(name, "", type, extract, label, noise, order) +qspec(name; type="real_scalar", extract="value", label=name, noise=0.0, order=1, class="") = + QuantitySpec(name, "", type, extract, label, noise, order, class) golden_val(name; value_type="real", value_real=nothing, value_int=nothing, value_text=nothing, - rtol=1e-6, atol=0.0, class="physics_converged", basis="measured", drift=NaN, spread=NaN, at="") = + rtol=1e-6, atol=0.0, class="physics_converged", basis="class-default (provisional)", drift=NaN, spread=NaN, at="") = GoldenValue(name, value_type, value_real, value_int, value_text, rtol, atol, class, basis, drift, spread, at) qtuple(; value_real=nothing, value_int=nothing, value_text=nothing, value_type="real", label="q") = @@ -35,49 +35,53 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val @testset "scalar real comparison" begin g = golden_val("x"; value_real=1.0, rtol=1e-6) - passed, dev, detail = compare_to_golden(qtuple(value_real=1.0 + 5e-7), g) + passed, dev, detail = compare_to_golden(qtuple(; value_real=1.0 + 5e-7), g) @test passed @test dev ≈ 5e-7 rtol = 1e-3 @test isempty(detail) - passed, dev, detail = compare_to_golden(qtuple(value_real=1.0 + 1e-5), g) + passed, dev, detail = compare_to_golden(qtuple(; value_real=1.0 + 1e-5), g) @test !passed @test dev ≈ 1e-5 rtol = 1e-3 @test !isempty(detail) # Zero golden value: rtol contributes nothing, deviation is reported absolute. g0 = golden_val("x"; value_real=0.0, rtol=1e-6, atol=0.0) - passed, dev, _ = compare_to_golden(qtuple(value_real=1e-12), g0) + passed, dev, _ = compare_to_golden(qtuple(; value_real=1e-12), g0) @test !passed @test dev ≈ 1e-12 g0a = golden_val("x"; value_real=0.0, rtol=1e-6, atol=1e-9) - @test compare_to_golden(qtuple(value_real=1e-12), g0a)[1] + @test compare_to_golden(qtuple(; value_real=1e-12), g0a)[1] - # Non-finite tolerances mean "recorded, never judged": must pass even on a zero golden - # value, where atol + rtol*|gold| would otherwise evaluate to Inf + NaN. + # Non-gating classes are never judged, even on a zero golden value where Inf*0 = NaN. gd = golden_val("x"; value_real=0.0, rtol=Inf, atol=Inf, class="diagnostic") - @test compare_to_golden(qtuple(value_real=42.0), gd)[1] + @test compare_to_golden(qtuple(; value_real=42.0), gd)[1] + # Finite tolerances on a non-gating class do not make it gate either. + @test compare_to_golden(qtuple(; value_real=42.0), golden_val("x"; value_real=1.0, class="unconverged"))[1] - @test !compare_to_golden(qtuple(value_real=nothing), g)[1] + @test !compare_to_golden(qtuple(; value_real=nothing), g)[1] + # A NaN run value fails a finite gold rather than slipping through a false comparison. + passed, dev, _ = compare_to_golden(qtuple(; value_real=NaN), g) + @test !passed && isnan(dev) end @testset "integer and token comparison" begin # Topological integers gate on exact equality no matter how loose the rtol. gt = golden_val("n"; value_type="integer", value_int=5, rtol=10.0, class="topological") - @test compare_to_golden(qtuple(value_int=5, value_type="integer"), gt)[1] - @test !compare_to_golden(qtuple(value_int=6, value_type="integer"), gt)[1] + @test compare_to_golden(qtuple(; value_int=5, value_type="integer"), gt)[1] + @test !compare_to_golden(qtuple(; value_int=6, value_type="integer"), gt)[1] # Non-topological integers use the tolerance like a real. gi = golden_val("n"; value_type="integer", value_int=100, rtol=0.05, class="physics_converged") - @test compare_to_golden(qtuple(value_int=103, value_type="integer"), gi)[1] - @test !compare_to_golden(qtuple(value_int=110, value_type="integer"), gi)[1] + @test compare_to_golden(qtuple(; value_int=103, value_type="integer"), gi)[1] + @test !compare_to_golden(qtuple(; value_int=110, value_type="integer"), gi)[1] # Tokens are discrete: deviation 0.0 on match, 1.0 (not NaN) on mismatch so it sorts as # a real failure. gk = golden_val("integ"; value_type="token", value_text="riccati", class="topological") - passed, dev, _ = compare_to_golden(qtuple(value_text="riccati", value_type="token"), gk) + passed, dev, _ = compare_to_golden(qtuple(; value_text="riccati", value_type="token"), gk) @test passed && dev == 0.0 - passed, dev, detail = compare_to_golden(qtuple(value_text="galerkin", value_type="token"), gk) + passed, dev, detail = compare_to_golden(qtuple(; value_text="galerkin", value_type="token"), gk) @test !passed && dev == 1.0 @test occursin("riccati", detail) && occursin("galerkin", detail) end @@ -85,48 +89,60 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val @testset "array worst-element path" begin g = golden_val("v"; value_type="json_array", value_text="[1.0,2.0,4.0]", rtol=1e-6) - passed, dev, detail = compare_to_golden(qtuple(value_text="[1.0000000001,2.0,4.0]", value_type="json_array"), g) + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0000000001,2.0,4.0]", value_type="json_array"), g) @test passed @test dev ≈ 1e-10 rtol = 1e-3 @test isempty(detail) # One element out of tolerance fails the whole array and the detail names it. - passed, dev, detail = compare_to_golden(qtuple(value_text="[1.0,2.002,4.0]", value_type="json_array"), g) + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0,2.002,4.0]", value_type="json_array"), g) @test !passed @test dev ≈ 1e-3 rtol = 1e-3 @test occursin("worst element 2", detail) # The reported deviation is the worst RELATIVE element, not the first failing one. - passed, dev, detail = compare_to_golden(qtuple(value_text="[1.001,2.0,4.02]", value_type="json_array"), g) + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.001,2.0,4.02]", value_type="json_array"), g) @test !passed @test dev ≈ 5e-3 rtol = 1e-3 @test occursin("worst element 3", detail) # A zero golden element is judged absolutely; atol=0 makes any movement on it fail. gz = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=1e-6, atol=0.0) - @test !compare_to_golden(qtuple(value_text="[1.0e-9,1.0]", value_type="json_array"), gz)[1] + @test !compare_to_golden(qtuple(; value_text="[1.0e-9,1.0]", value_type="json_array"), gz)[1] gza = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=1e-6, atol=1e-8) - @test compare_to_golden(qtuple(value_text="[1.0e-9,1.0]", value_type="json_array"), gza)[1] + @test compare_to_golden(qtuple(; value_text="[1.0e-9,1.0]", value_type="json_array"), gza)[1] # Non-finite tolerances never judge, including zero golden elements (the Inf*0 = NaN guard). gdz = golden_val("v"; value_type="json_array", value_text="[0.0,1.0]", rtol=Inf, atol=Inf, class="diagnostic") - @test compare_to_golden(qtuple(value_text="[5.0,99.0]", value_type="json_array"), gdz)[1] + @test compare_to_golden(qtuple(; value_text="[5.0,99.0]", value_type="json_array"), gdz)[1] # Complex encoding: [re, im] pairs go through the same worst-element machinery. gc = golden_val("v"; value_type="json_array", value_text="[[1.0,0.5],[2.0,-1.0]]", rtol=1e-6) - @test compare_to_golden(qtuple(value_text="[[1.0,0.5],[2.0,-1.0]]", value_type="json_array"), gc)[1] - passed, _, detail = compare_to_golden(qtuple(value_text="[[1.0,0.5],[2.0,-1.01]]", value_type="json_array"), gc) + @test compare_to_golden(qtuple(; value_text="[[1.0,0.5],[2.0,-1.0]]", value_type="json_array"), gc)[1] + passed, _, detail = compare_to_golden(qtuple(; value_text="[[1.0,0.5],[2.0,-1.01]]", value_type="json_array"), gc) @test !passed @test occursin("worst element 2", detail) - passed, _, detail = compare_to_golden(qtuple(value_text="[1.0,2.0]", value_type="json_array"), g) + passed, _, detail = compare_to_golden(qtuple(; value_text="[1.0,2.0]", value_type="json_array"), g) @test !passed @test occursin("length", detail) + + # A NaN element fails and is the element the detail names; NaN on both sides is "no result" + # in both runs and matches. + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0,NaN,4.0]", value_type="json_array"), g) + @test !passed && isinf(dev) + @test occursin("worst element 2", detail) + gn = golden_val("v"; value_type="json_array", value_text="[1.0,NaN]", rtol=1e-6) + @test compare_to_golden(qtuple(; value_text="[1.0,NaN]", value_type="json_array"), gn)[1] + + # A nested row that changed length fails instead of being truncated by zip. + gr = golden_val("m"; value_type="json_array", value_text="[[1.0,2.0],[3.0,4.0]]", rtol=1e-6) + @test !compare_to_golden(qtuple(; value_text="[[1.0,2.0],[3.0]]", value_type="json_array"), gr)[1] end @testset "type-change handling" begin g = golden_val("x"; value_real=1.0) - passed, dev, detail = compare_to_golden(qtuple(value_text="[1.0]", value_type="json_array"), g) + passed, dev, detail = compare_to_golden(qtuple(; value_text="[1.0]", value_type="json_array"), g) @test !passed && isnan(dev) @test occursin("type changed", detail) @@ -151,51 +167,112 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val @test vals2["x"].tolerance_basis == "measured" @test vals2["x"].plateau_drift == 1e-8 && vals2["x"].platform_spread == 2e-7 - # Missing extractions, checksums, and quantities without a spec never become entries. + # A change of the case-declared class resets measurements, like a type change. + prior3 = Dict("x" => golden_val("x"; value_real=1.0, rtol=3e-7, basis="measured", drift=1e-8)) + vals4 = @test_logs (:warn, r"class changed") build_golden_values( + [extracted_q("x"; value_real=1.0)], [qspec("x"; class="equilibrium_scalar")], prior3) + @test vals4["x"].class == "equilibrium_scalar" + @test vals4["x"].tolerance_basis == "class-default (provisional)" + + # Missing extractions, checksums, runtimes, and quantities without a spec never become entries. vals3 = build_golden_values( [extracted_q("gone"; value_type="missing"), extracted_q("x"; value_text="abc123", value_type="checksum"), + extracted_q("rt"; value_real=12.5), extracted_q("unspecced"; value_real=1.0)], - [spec], nothing) + [spec, qspec("rt"; type="runtime")], nothing) @test isempty(vals3) end @testset "load-validation refusals" begin - write(golden_path("bad_class"), """ - [values.x] - class = "definitely_not_a_class" - rtol = 1.0e-6 - value = 1.0 - """) + write( + golden_path("bad_class"), + """ +[values.x] +class = "definitely_not_a_class" +rtol = 1.0e-6 +value = 1.0 +""" + ) @test_throws ErrorException load_golden("bad_class") # A gating entry with no finite rtol is malformed whether it was hand-edited or merged. - write(golden_path("no_rtol"), """ - [values.x] - class = "physics_converged" - value = 1.0 - """) + write( + golden_path("no_rtol"), + """ +[values.x] +class = "physics_converged" +value = 1.0 +""" + ) @test_throws ErrorException load_golden("no_rtol") # An rtol below the recorded platform spread is a gate no second platform can pass; # enforced on every load so a hand edit cannot ship what save_golden refused to write. - write(golden_path("too_tight"), """ - [values.x] - class = "physics_converged" - rtol = 1.0e-9 - platform_spread = 1.0e-6 - value = 1.0 - """) + write( + golden_path("too_tight"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-9 +platform_spread = 1.0e-6 +value = 1.0 +""" + ) @test_throws ErrorException load_golden("too_tight") + # A non-finite atol on a gating class would make every comparison pass. + for bad in ("inf", "nan") + write( + golden_path("atol_$bad"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-6 +atol = $bad +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("atol_$bad") + end + + # An rtol below the measured plateau drift is a gate the converged answer itself fails. + write( + golden_path("below_drift"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-9 +plateau_drift = 1.0e-7 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("below_drift") + + # "measured" must be backed by a recorded measurement. + write( + golden_path("unbacked"), + """ +[values.x] +class = "physics_converged" +tolerance_basis = "measured" +rtol = 1.0e-6 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("unbacked") + # Non-gating classes are exempt: they are recorded, never judged. - write(golden_path("diag_inf"), """ - [values.rt] - class = "diagnostic" - rtol = inf - atol = inf - value = 12.5 - """) + write( + golden_path("diag_inf"), + """ +[values.rt] +class = "diagnostic" +rtol = inf +atol = inf +value = 12.5 +""" + ) loaded = load_golden("diag_inf") @test loaded !== nothing && !is_gating(loaded.values["rt"]) @@ -209,6 +286,8 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val @test_throws ErrorException save_golden(meta, Dict("empty" => golden_val("empty"))) @test_throws ErrorException save_golden(meta, Dict("tight" => golden_val("tight"; value_real=1.0, rtol=1e-9, spread=1e-6))) + @test_throws ErrorException save_golden(meta, + Dict("open" => golden_val("open"; value_real=1.0, rtol=1e-6, atol=Inf, spread=1e-7))) vals = Dict( "a" => golden_val("a"; value_real=2.5, rtol=1e-6, drift=1e-8, spread=1e-7, at="mpsi=512"), @@ -230,38 +309,59 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val @test infer_class(qspec("msing"; type="int_scalar")) == "topological" @test infer_class(qspec("integrator"; type="token", extract="toml_key:ForceFreeStates.integrator")) == "topological" @test infer_class(qspec("q0")) == "equilibrium_scalar" + @test infer_class(qspec("psio")) == "equilibrium_scalar" @test infer_class(qspec("et1_re")) == "physics_converged" + # A class declared in the case file wins over inference, and must be a real class. + @test infer_class(qspec("ntv_psi_nsteps"; type="int_scalar", class="diagnostic")) == "diagnostic" + @test_throws ErrorException infer_class(qspec("x"; class="gating_optional")) end @testset "golden check counting and crash classification" begin db = open_database(joinpath(mktempdir(), "check.sqlite")) - case = CaseSpec("count_case", "synthetic", "", [ + case = CaseSpec("count_case", "synthetic", "", + [ qspec("a"), qspec("rt"; type="runtime", extract=""), qspec("c"; type="int_scalar"), - qspec("d"), + qspec("d"; class="diagnostic"), qspec("e"), - qspec("f")], "example", Dict{String,Any}()) + qspec("f"), + qspec("demoted")], "example", Dict{String,Any}()) meta = GoldenMeta("count_case", 1, "2026-08-31", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4) - save_golden(meta, Dict( - "a" => golden_val("a"; value_real=1.0, rtol=1e-6), - "c" => golden_val("c"; value_type="integer", value_int=5, rtol=0.0, class="topological"), - "d" => golden_val("d"; value_real=3.0, rtol=Inf, atol=Inf, class="diagnostic"), - "f" => golden_val("f"; value_real=9.0, rtol=1e-6), - "orphan" => golden_val("orphan"; value_real=7.0, rtol=1e-6))) - - # a passes, c fails (5 → 6), d moves wildly but is diagnostic, e has no golden, - # f is golden-pinned but missing from the run, orphan has no spec left in the case. - store_run(db, "hash1", "hash1", "", "", "count_case", 1.0, [ - extracted_q("a"; value_real=1.0 + 1e-8), - extracted_q("rt"; value_real=33.0), - extracted_q("c"; value_int=6, value_type="integer"), - extracted_q("d"; value_real=300.0), - extracted_q("e"; value_real=1.0)]) + save_golden( + meta, + Dict( + "a" => golden_val("a"; value_real=1.0, rtol=1e-6), + "c" => golden_val("c"; value_type="integer", value_int=5, rtol=0.0, class="topological"), + "d" => golden_val("d"; value_real=3.0, rtol=Inf, atol=Inf, class="diagnostic"), + "f" => golden_val("f"; value_real=9.0, rtol=1e-6), + "demoted" => golden_val("demoted"; value_real=2.0, rtol=Inf, atol=Inf, class="unconverged"), + "orphan" => golden_val("orphan"; value_real=7.0, rtol=1e-6)) + ) + + # a passes, c fails (5 → 6), d moves wildly but is declared diagnostic, e has no golden, + # f is golden-pinned but missing from the run, demoted is non-gating in the golden file only, + # orphan has no spec left in the case. + store_run( + db, + "hash1", + "hash1", + "", + "", + "count_case", + 1.0, + [ + extracted_q("a"; value_real=1.0 + 1e-8), + extracted_q("rt"; value_real=33.0), + extracted_q("c"; value_int=6, value_type="integer"), + extracted_q("d"; value_real=300.0), + extracted_q("e"; value_real=1.0), + extracted_q("demoted"; value_real=50.0)] + ) s = report_golden_check(db, case, "hash1") @test s.n_pass == 1 - @test s.n_fail == 3 # c mismatch + f missing from run + orphaned golden entry + @test s.n_fail == 4 # c mismatch + f missing + class demoted outside the case + orphan @test s.n_informational == 1 @test s.n_untracked == 1 # e only: an unpinned runtime is structurally un-goldenable @test s.n_run_failed == 0 @@ -277,6 +377,47 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val nocase = CaseSpec("never_pinned", "synthetic", "", [qspec("a")], "example", Dict{String,Any}()) s = report_golden_check(db, nocase, "hash1") @test s == (n_pass=0, n_fail=0, n_untracked=0, n_informational=0, n_run_failed=0) + + # A golden file holding only non-gating entries gates nothing, and the counts say so. + diagcase = CaseSpec("diag_only", "synthetic", "", [qspec("d"; class="diagnostic")], "example", Dict{String,Any}()) + save_golden(GoldenMeta("diag_only", 1, "2026-08-31", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4), + Dict("d" => golden_val("d"; value_real=3.0, rtol=Inf, atol=Inf, class="diagnostic"))) + store_run(db, "hash3", "hash3", "", "", "diag_only", 1.0, [extracted_q("d"; value_real=30.0)]) + s = report_golden_check(db, diagcase, "hash3") + @test s.n_pass + s.n_fail == 0 && s.n_informational == 1 close_database(db) end + + @testset "re-pin reporting" begin + old = golden_val("a"; value_real=1.0, rtol=1e-6) + @test describe_golden_change(old, golden_val("a"; value_real=1.0, rtol=1e-6)) === nothing + @test !occursin("EXCEEDS", describe_golden_change(old, golden_val("a"; value_real=1.0 + 1e-7))) + @test occursin("EXCEEDS", describe_golden_change(old, golden_val("a"; value_real=1.1))) + # Integer and token moves print their values, not just "changed". + oi = golden_val("n"; value_type="integer", value_int=5, rtol=0.0, class="topological") + line = describe_golden_change(oi, golden_val("n"; value_type="integer", value_int=4, rtol=0.0, class="topological")) + @test occursin("5 → 4", line) && occursin("EXCEEDS", line) + ok = golden_val("k"; value_type="token", value_text="riccati", class="topological") + @test occursin("riccati → galerkin", describe_golden_change(ok, golden_val("k"; value_type="token", value_text="galerkin", class="topological"))) + # Moves on a non-gating entry are shown but never flagged. + od = golden_val("d"; value_real=1.0, rtol=Inf, atol=Inf, class="diagnostic") + @test !occursin("EXCEEDS", describe_golden_change(od, golden_val("d"; value_real=9.0, rtol=Inf, atol=Inf, class="diagnostic"))) + end + + @testset "update refuses uncommitted source" begin + repo = mktempdir() + run(`git -C $repo init -q`) + mkpath(joinpath(repo, "golden")) + write(joinpath(repo, "src.jl"), "x = 1\n") + write(joinpath(repo, "golden", "case.toml"), "a = 1\n") + run(`git -C $repo add -A`) + run(`git -C $repo -c user.name=t -c user.email=t@t commit -q -m init`) + @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) + # Rewriting a golden file is the update itself, and untracked files are not source. + write(joinpath(repo, "golden", "case.toml"), "a = 2\n") + write(joinpath(repo, "scratch.md"), "notes\n") + @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) + write(joinpath(repo, "src.jl"), "x = 2\n") + @test occursin("src.jl", uncommitted_changes(repo, joinpath(repo, "golden"))) + end end From fb01d888b8e3cd8c25fcdeb8e2607b1c1f11f43e Mon Sep 17 00:00:00 2001 From: d-burg Date: Wed, 23 Sep 2026 16:22:08 -0400 Subject: [PATCH 18/30] Regression - TEST - Regenerate the Fitzpatrick golden on the branch with exact scan axes Values are unchanged; the scan axes and tau move to topological, the runtime pin is dropped, and the provenance commit is now reachable from the branch. Co-Authored-By: Claude Opus 5.5 --- .../golden/slayer_dels_fitzpatrick.toml | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/regression-harness/golden/slayer_dels_fitzpatrick.toml b/regression-harness/golden/slayer_dels_fitzpatrick.toml index ddb9ce848..cd528b434 100644 --- a/regression-harness/golden/slayer_dels_fitzpatrick.toml +++ b/regression-harness/golden/slayer_dels_fitzpatrick.toml @@ -1,14 +1,14 @@ [meta] -blas_threads = 2 +blas_threads = 8 case = "slayer_dels_fitzpatrick" -commit = "af3e4492" -generated_at = "2026-08-18" -golden_version = 1 +commit = "67a2c40cb" +generated_at = "2026-09-23" +golden_version = 2 julia_version = "1.11.6" manifest_sha = "7e5c34ad1722fa6addf94f7387b972fa8ac17ff86166793819b158496d495132" -nthreads = 2 +nthreads = 8 os_arch = "arm64-apple-darwin24.0.0" -reason = "Initial pin: validated against Fitzpatrick 2023 figs 6.2/6.3 at eight contour points" +reason = "Regenerate on the branch after the second adversarial review: scan axes and tau now topological, runtime no longer pinned; values unchanged from the Fitzpatrick-validated state" [values.dels_db_im] atol = 0.0 @@ -28,32 +28,24 @@ value_type = "json_array" [values.p_hat] atol = 0.0 -class = "physics_converged" -rtol = 1.0e-6 +class = "topological" +rtol = 0.0 tolerance_basis = "class-default (provisional)" value_text = "[0.02,0.02,0.02,0.02,0.02,0.5,0.5,0.5,0.5,0.5,1.0,1.0,1.0,1.0,1.0,2.0,2.0,2.0,2.0,2.0,4.0,4.0,4.0,4.0,4.0]" value_type = "json_array" [values.q_hat] atol = 0.0 -class = "physics_converged" -rtol = 1.0e-6 +class = "topological" +rtol = 0.0 tolerance_basis = "class-default (provisional)" value_text = "[0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0,0.02,0.5,1.0,2.0,4.0]" value_type = "json_array" -[values.runtime] -atol = +inf -class = "diagnostic" -rtol = +inf -tolerance_basis = "class-default (provisional)" -value = 1.2057960033416748 -value_type = "real" - [values.tau] atol = 0.0 -class = "physics_converged" -rtol = 1.0e-6 +class = "topological" +rtol = 0.0 tolerance_basis = "class-default (provisional)" value = 1.0 value_type = "real" From 4f45706b76218856b64a5379c55cc060757eb6af Mon Sep 17 00:00:00 2001 From: d-burg Date: Wed, 23 Sep 2026 16:22:17 -0400 Subject: [PATCH 19/30] Regression - MINOR - Say a dropped runtime pin is intentional in the golden update report Co-Authored-By: Claude Opus 5.5 --- regression-harness/src/golden.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 5dad40149..2da6b6837 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -495,8 +495,11 @@ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash: println() println("Golden update: $(case_spec.name) (v$(previous === nothing ? 0 : previous.meta.golden_version) → v$(meta.golden_version))") if existing !== nothing + runtime_names = Set(s.name for s in case_spec.quantities if s.type == "runtime") for name in sort(collect(keys(existing))) - if !haskey(values, name) + if name in runtime_names + println(@sprintf(" %-34s REMOVED — runtimes are no longer pinned", name)) + elseif !haskey(values, name) println( @sprintf( " %-34s REMOVED — extraction returned missing (renamed h5 path?) or the quantity left the case. This deletes its gate; confirm it is intentional.", From ea060a851504c542c73a74c2f24ebe7c634780f7 Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 11:04:42 -0400 Subject: [PATCH 20/30] Regression - BUGFIX - Drop stale measured evidence on a re-pin and show atol and tolerance basis in golden checks A re-pin whose value moves beyond the old gating tolerance no longer carries the old plateau_drift/platform_spread forward; the entry becomes provisional with rtol/atol no looser than before. The --check table now reports atol and whether each gating tolerance is measured or provisional, marks absolute deviations, and one is_pinnable predicate replaces three inline copies. Co-Authored-By: Claude Opus 5.5 --- regression-harness/src/golden.jl | 74 +++++++++++++++++++---------- regression-harness/test/runtests.jl | 26 ++++++++-- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 2da6b6837..c82df1962 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -64,7 +64,8 @@ One quantity's pinned value and the terms it is judged by. - `rtol` / `atol` — pass when `|x - gold| <= atol + rtol*|gold|` - `class` — one of `TOLERANCE_CLASSES` - `tolerance_basis` — how the tolerance was arrived at: "measured" (requires a recorded - `plateau_drift` or `platform_spread`) or "class-default (provisional)" + `plateau_drift` or `platform_spread`), "class-default (provisional)", or "provisional" (no + measurement, tolerance kept tighter than the class default after a re-pin dropped the evidence) - `plateau_drift` — relative movement over the final refinement of the convergence scan; NaN when not measured - `platform_spread` — relative disagreement between reference platforms; NaN when not measured @@ -88,25 +89,35 @@ end is_gating(g::GoldenValue) = g.class in GATING_CLASSES +""" +True when a tolerance is not backed by a recorded measurement, i.e. any basis other than "measured". +""" +is_provisional(g::GoldenValue) = g.tolerance_basis != "measured" + +""" +Runtimes are wall-clock and checksums have no notion of "close", so neither is ever pinned. +""" +is_pinnable(spec::QuantitySpec) = spec.type != "runtime" && spec.extract != "checksum" + """ Refuse an entry that would gate more quietly than its evidence allows. Run on every load as well as every save, so a hand edit or a merge taking the wrong side is caught as surely as a bad write. """ -function validate_golden_value(g::GoldenValue, where::AbstractString) - g.class in TOLERANCE_CLASSES || error("$where: quantity '$(g.name)' has unknown class '$(g.class)'") +function validate_golden_value(g::GoldenValue, context::AbstractString) + g.class in TOLERANCE_CLASSES || error("$context: quantity '$(g.name)' has unknown class '$(g.class)'") if is_gating(g) (isfinite(g.rtol) && g.rtol >= 0 && isfinite(g.atol) && g.atol >= 0) || - error("$where: gating quantity '$(g.name)' needs a finite, non-negative rtol and atol (got rtol=$(g.rtol), atol=$(g.atol))") + error("$context: gating quantity '$(g.name)' needs a finite, non-negative rtol and atol (got rtol=$(g.rtol), atol=$(g.atol))") for (label, m) in (("platform_spread", g.platform_spread), ("plateau_drift", g.plateau_drift)) isfinite(m) && g.rtol < m && error( - "$where: quantity '$(g.name)' has rtol $(g.rtol) below its measured $label $m. " * + "$context: quantity '$(g.name)' has rtol $(g.rtol) below its measured $label $m. " * "Widen it deliberately, with the measurement recorded, or reduce the $label before pinning." ) end end g.tolerance_basis == "measured" && !isfinite(g.plateau_drift) && !isfinite(g.platform_spread) && - error("$where: quantity '$(g.name)' claims a measured tolerance but records neither plateau_drift nor platform_spread") + error("$context: quantity '$(g.name)' claims a measured tolerance but records neither plateau_drift nor platform_spread") return nothing end @@ -353,26 +364,25 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St end rows = Vector{Vector{String}}() - n_pass = n_fail = n_untracked = n_informational = 0 + n_pass = n_fail = n_untracked = n_informational = n_provisional = 0 for spec in case_spec.quantities g = get(golden.values, spec.name, nothing) if g === nothing - # Runtimes and checksums are never pinned, so they do not count as untracked physics. - (spec.type == "runtime" || spec.extract == "checksum") && continue + is_pinnable(spec) || continue n_untracked += 1 continue end q_raw = get(quantities, spec.name, nothing) if q_raw === nothing - push!(rows, [spec.label, g.class, "MISSING", "—", "FAIL"]) + push!(rows, [spec.label, g.class, "MISSING", "—", "—", "—", "FAIL"]) n_fail += 1 continue end # The golden file's class must be the one the case declares, so a demotion cannot hide there. declared = infer_class(spec) if g.class != declared - push!(rows, [spec.label, g.class, "CLASS", "—", "** FAIL ** case declares $declared; regenerate goldens"]) + push!(rows, [spec.label, g.class, "CLASS", "—", "—", "—", "** FAIL ** case declares $declared; regenerate goldens"]) n_fail += 1 continue end @@ -396,21 +406,26 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St else n_fail += 1 end - dev = isnan(deviation) ? "—" : @sprintf("%.2e", deviation) - tol = isfinite(g.rtol) ? @sprintf("%.1e", g.rtol) : "—" - push!(rows, [spec.label, g.class, dev, tol, status * (isempty(detail) ? "" : " $detail")]) + # A scalar with a zero golden value is judged absolutely, so its deviation is marked as such. + zero_gold = (g.value_type == "real" && g.value_real == 0.0) || (g.value_type == "integer" && g.value_int == 0) + dev = isnan(deviation) ? "—" : @sprintf("%.2e", deviation) * (zero_gold ? " (abs)" : "") + rtol = isfinite(g.rtol) ? @sprintf("%.1e", g.rtol) : "—" + atol = isfinite(g.atol) ? @sprintf("%.1e", g.atol) : "—" + basis = !gating ? "—" : is_provisional(g) ? "provisional" : "measured" + gating && is_provisional(g) && (n_provisional += 1) + push!(rows, [spec.label, g.class, dev, rtol, atol, basis, status * (isempty(detail) ? "" : " $detail")]) end # An orphaned entry fails regardless of class: a renamed quantity must not silently drop its gate. spec_names = Set(spec.name for spec in case_spec.quantities) for name in sort(collect(keys(golden.values))) name in spec_names && continue - push!(rows, [name, golden.values[name].class, "ORPHANED", "—", + push!(rows, [name, golden.values[name].class, "ORPHANED", "—", "—", "—", "** FAIL ** no matching quantity in the case — renamed or removed? Regenerate goldens."]) n_fail += 1 end - header = ["Quantity", "Class", "Deviation", "rtol", "Status"] + header = ["Quantity", "Class", "Deviation", "rtol", "atol", "Basis", "Status"] widths = [length(h) for h in header] for row in rows, i in eachindex(row) widths[i] = max(widths[i], length(row[i])) @@ -439,6 +454,10 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St n_informational > 0 && push!(parts, "$n_informational informational") n_untracked > 0 && push!(parts, "$n_untracked untracked") println("Summary: ", join(parts, ", ")) + n_provisional > 0 && println( + " $n_provisional gating quantity/quantities are judged against provisional class-default tolerances, " * + "not a measured plateau drift or platform spread." + ) println() return (n_pass=n_pass, n_fail=n_fail, n_untracked=n_untracked, n_informational=n_informational, n_run_failed=0) end @@ -495,10 +514,10 @@ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash: println() println("Golden update: $(case_spec.name) (v$(previous === nothing ? 0 : previous.meta.golden_version) → v$(meta.golden_version))") if existing !== nothing - runtime_names = Set(s.name for s in case_spec.quantities if s.type == "runtime") + unpinnable = Set(s.name for s in case_spec.quantities if !is_pinnable(s)) for name in sort(collect(keys(existing))) - if name in runtime_names - println(@sprintf(" %-34s REMOVED — runtimes are no longer pinned", name)) + if name in unpinnable + println(@sprintf(" %-34s REMOVED — runtimes and checksums are not pinned", name)) elseif !haskey(values, name) println( @sprintf( @@ -519,7 +538,7 @@ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash: end path = save_golden(meta, values) println("Wrote $path ($(length(values)) quantities)") - provisional = count(g -> startswith(g.tolerance_basis, "class-default"), Base.values(values)) + provisional = count(g -> is_gating(g) && is_provisional(g), Base.values(values)) provisional > 0 && println( " $provisional quantity/quantities still carry provisional class-default tolerances — " * "replace them with measured plateau drift and platform spread before relying on this as a gate." @@ -558,7 +577,9 @@ Build golden entries from a set of freshly extracted quantities. `existing` carries forward the tolerance and evidence already recorded for a quantity, so regenerating values after a physics change does not silently reset hard-won measurements to -provisional defaults. A change of value type or of the case-declared class invalidates them. +provisional defaults. A change of value type or of the case-declared class invalidates them. So +does a move beyond the old gating tolerance: the evidence was measured on a different value, so it +is dropped and the entry becomes provisional, with a tolerance no looser than before. """ function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector{QuantitySpec}, existing::Union{Dict{String,GoldenValue},Nothing}) @@ -568,8 +589,7 @@ function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector eq.value_type == "missing" && continue spec = get(spec_by_name, eq.name, nothing) spec === nothing && continue - # Checksums have no notion of "close" and runtimes are wall-clock, so neither is pinned. - (eq.value_type == "checksum" || spec.type == "runtime") && continue + is_pinnable(spec) || continue class = infer_class(spec) prior = existing === nothing ? nothing : get(existing, eq.name, nothing) if prior !== nothing && prior.value_type != eq.value_type @@ -579,10 +599,16 @@ function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector @warn "Golden '$(eq.name)': class changed $(prior.class) → $class; resetting tolerances to provisional" prior = nothing end + defaults = CLASS_DEFAULT_TOLERANCE[class] + run_value = (value_real=eq.value_real, value_int=eq.value_int, value_text=eq.value_text, value_type=eq.value_type) if prior === nothing - defaults = CLASS_DEFAULT_TOLERANCE[class] rtol, atol, basis = defaults.rtol, defaults.atol, "class-default (provisional)" drift, spread, at = NaN, NaN, "" + elseif is_gating(prior) && !compare_to_golden(run_value, prior)[1] + @warn "Golden '$(eq.name)': value moved beyond its old tolerance; dropping its recorded evidence and marking it provisional" + rtol, atol = min(prior.rtol, defaults.rtol), min(prior.atol, defaults.atol) + basis = (rtol, atol) == (defaults.rtol, defaults.atol) ? "class-default (provisional)" : "provisional" + drift, spread, at = NaN, NaN, "" else rtol, atol, basis = prior.rtol, prior.atol, prior.tolerance_basis drift, spread, at = prior.plateau_drift, prior.platform_spread, prior.converged_at diff --git a/regression-harness/test/runtests.jl b/regression-harness/test/runtests.jl index 49666b601..a36ca5817 100644 --- a/regression-harness/test/runtests.jl +++ b/regression-harness/test/runtests.jl @@ -158,15 +158,29 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val @test vals["x"].tolerance_basis == "class-default (provisional)" @test isnan(vals["x"].plateau_drift) && isnan(vals["x"].platform_spread) - # Same type carries the prior's hard-won measurements forward unchanged. + # Same type, moved within the old tolerance: the prior's measurements carry forward unchanged. prior2 = Dict("x" => golden_val("x"; value_real=1.0, rtol=3e-7, atol=1e-12, basis="measured", drift=1e-8, spread=2e-7, at="mpsi=1024")) - vals2 = build_golden_values([extracted_q("x"; value_real=1.5)], [spec], prior2) - @test vals2["x"].value_real == 1.5 + vals2 = build_golden_values([extracted_q("x"; value_real=1.0 + 1e-7)], [spec], prior2) + @test vals2["x"].value_real == 1.0 + 1e-7 @test vals2["x"].rtol == 3e-7 && vals2["x"].atol == 1e-12 @test vals2["x"].tolerance_basis == "measured" @test vals2["x"].plateau_drift == 1e-8 && vals2["x"].platform_spread == 2e-7 + # A move beyond the old tolerance drops the evidence, and the tolerance never loosens. + vals5 = @test_logs (:warn, r"beyond its old tolerance") build_golden_values( + [extracted_q("x"; value_real=1.5)], [spec], prior2) + @test vals5["x"].value_real == 1.5 + @test vals5["x"].rtol == 3e-7 && vals5["x"].atol == 0.0 + @test vals5["x"].tolerance_basis == "provisional" && is_provisional(vals5["x"]) + @test isnan(vals5["x"].plateau_drift) && isnan(vals5["x"].platform_spread) && isempty(vals5["x"].converged_at) + # A measured tolerance wider than the class default is tightened back to the default. + prior5 = Dict("x" => golden_val("x"; value_real=1.0, rtol=1e-4, basis="measured", drift=5e-5)) + vals6 = @test_logs (:warn, r"beyond its old tolerance") build_golden_values( + [extracted_q("x"; value_real=1.5)], [spec], prior5) + @test vals6["x"].rtol == CLASS_DEFAULT_TOLERANCE["physics_converged"].rtol + @test vals6["x"].tolerance_basis == "class-default (provisional)" + # A change of the case-declared class resets measurements, like a type change. prior3 = Dict("x" => golden_val("x"; value_real=1.0, rtol=3e-7, basis="measured", drift=1e-8)) vals4 = @test_logs (:warn, r"class changed") build_golden_values( @@ -316,6 +330,12 @@ value = 12.5 @test_throws ErrorException infer_class(qspec("x"; class="gating_optional")) end + @testset "is_pinnable" begin + @test is_pinnable(qspec("a")) + @test !is_pinnable(qspec("rt"; type="runtime")) + @test !is_pinnable(qspec("h"; extract="checksum")) + end + @testset "golden check counting and crash classification" begin db = open_database(joinpath(mktempdir(), "check.sqlite")) case = CaseSpec("count_case", "synthetic", "", From 763009981c466b206db43d0111bd60558f1beeec Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 11:04:42 -0400 Subject: [PATCH 21/30] Regression - DOCS - Document golden-value mode and drop the claim that CI runs it Co-Authored-By: Claude Opus 5.5 --- docs/development/regression-harness.md | 16 +++++++++++++++- regression-harness/regress.jl | 5 +++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md index e9246c962..0f61d4773 100644 --- a/docs/development/regression-harness.md +++ b/docs/development/regression-harness.md @@ -140,7 +140,21 @@ re-run once — those are exactly the entries whose provenance cannot be establi If the two refs in a comparison ran under different thread counts, the report flags it. +## Golden values + +`--refs` comparisons say whether a number changed; golden values say whether it still matches a committed, reviewed reference. Each case may have a golden file at `regression-harness/golden/.toml` holding every pinned quantity with its `rtol`, `atol`, tolerance class and the evidence behind the tolerance. + +```bash +regress --check --cases slayer_dels_fitzpatrick # working tree vs the committed goldens +regress --update-golden --cases slayer_dels_fitzpatrick --reason "..." # re-pin from a fresh run +``` + +- **Classes** come from the case file's `class` key (else `infer_class` in `src/golden.jl`), never from the golden file: `topological` (exact), `equilibrium_scalar`, `physics_converged` gate; `diagnostic` and `unconverged` are reported only. +- **Tolerances** are either the provisional class default (`tolerance_basis = "class-default (provisional)"`) or `"measured"`, which requires a recorded `plateau_drift` or `platform_spread` no larger than `rtol`. No harness tool measures these yet, so every committed golden currently gates on provisional defaults, and `--check` labels those rows `provisional`. +- **Re-pinning** requires `--cases` and `--reason`, refuses a working tree with uncommitted tracked changes, and prints an old → new line per moved quantity, flagging any move the old tolerance would have failed. Such an entry loses its recorded evidence and becomes provisional, with a tolerance no looser than before. +- A failing `--check` is fixed by explaining the physics or fixing the regression, never by widening `rtol` or `atol`. The unit tests for this logic run with `julia --project=regression-harness regression-harness/test/runtests.jl` and need no GPEC build. + ## Exit status - `0` — every run completed (and, with `--fail-on-change`, nothing changed) -- `1` — a run failed, or a quantity changed under `--fail-on-change` +- `1` — a run failed, a quantity changed under `--fail-on-change`, or a `--check` golden comparison failed or gated nothing diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 356ddfeb5..1f097e7c4 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -132,7 +132,7 @@ Options: --fail-on-change Exit non-zero if any tracked quantity changed (for CI use; a failed run always exits non-zero regardless) --check Run the working tree and compare against the committed golden values - (regression-harness/golden/.toml). This is the mode CI gates on. + (regression-harness/golden/.toml). No CI workflow runs it yet. --update-golden Regenerate the golden values from a fresh run. Requires --cases and --reason, and refuses a working tree with uncommitted changes. --reason "..." Why the goldens changed. Mandatory with --update-golden: a golden @@ -149,7 +149,8 @@ Golden values: plateau drift and cross-platform spread; until measured they are provisional class defaults and say so. A quantity's class is declared in its case file (`class = ...`) or inferred, never set in the golden file. A failing --check is fixed by explaining the physics or - fixing the regression, not by loosening rtol. + fixing the regression, not by loosening rtol. A re-pin that moves a value beyond its old + tolerance drops the recorded evidence and marks the entry provisional. Exit status: 0 all runs completed (and, with --fail-on-change, nothing changed) From fc1b66c999b2a02bc84454415799130457a6fc58 Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 11:04:43 -0400 Subject: [PATCH 22/30] Regression - MINOR - Declare the probe grid topological and trim golden case narrative Co-Authored-By: Claude Opus 5.5 --- .../cases/slayer_dels_fitzpatrick.toml | 19 ++++++------------- .../cases/slayer_delta_probe.toml | 4 +++- .../golden/slayer_dels_fitzpatrick.toml | 2 +- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/regression-harness/cases/slayer_dels_fitzpatrick.toml b/regression-harness/cases/slayer_dels_fitzpatrick.toml index 7e39e48c1..0cede1614 100644 --- a/regression-harness/cases/slayer_dels_fitzpatrick.toml +++ b/regression-harness/cases/slayer_dels_fitzpatrick.toml @@ -1,17 +1,10 @@ # Regression case: GPEC's del_s Riccati layer-width solver against Fitzpatrick, "Tearing Mode -# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3 (p. 6-10). -# -# Verified by hand at eight points against the book's published contour bands, e.g. (Q,P) = -# (4.0, 0.5) -> Re 2.008 against the 1.9-2.1 contours, (0.5, 4.0) -> Re 1.537 against the 1.5 -# contour; the magnitudes also match its table 6.2 reactor estimates (|delta_s| 2.95-4.69 mm) at -# DIII-D-like parameters. The golden file pins GPEC's output at that validated state to a -# provisional class-default tolerance: a failure means the validated numbers moved, and any re-pin -# must be re-checked against the figures, since the gate itself is not the contour-band agreement. -# -# Pure ODE: no equilibrium, no geqdsk, no kinetic file. Sub-second, so it is fast-tier eligible, and -# it carries no equilibrium dependence -- its golden values survive equilibrium-side re-baselining. -# Complements slayer_delta_probe, which pins the dispersion solve Delta(Q); this pins the width -# solve delta_s/d_beta over (Q_hat, P_hat). +# Dynamics in Tokamak Plasmas" (IOP 2023), figures 6.2 and 6.3 (p. 6-10), over (Q_hat, P_hat). +# Hand-checked at eight points against the published contour bands, e.g. (Q,P) = (4.0, 0.5) -> +# Re 2.008 against the 1.9-2.1 contours. The golden pins GPEC's output at that validated state on +# a provisional tolerance, so any re-pin must be re-checked against the figures. +# Pure ODE, no equilibrium or kinetic input: sub-second and fast-tier eligible. Complements +# slayer_delta_probe, which pins the dispersion solve Delta(Q). [case] name = "slayer_dels_fitzpatrick" description = "del_s Riccati layer width vs Fitzpatrick 2023 figs 6.2/6.3 over (Q_hat, P_hat)" diff --git a/regression-harness/cases/slayer_delta_probe.toml b/regression-harness/cases/slayer_delta_probe.toml index 41a7d8798..41645a262 100644 --- a/regression-harness/cases/slayer_delta_probe.toml +++ b/regression-harness/cases/slayer_delta_probe.toml @@ -3,7 +3,7 @@ # SLAYER deck's own 2/1-surface layer parameters, quoted verbatim in the runner's computed # script. Fixed Q values are refinement-stable, unlike the AMR scan samples in gpec.h5, so # this pins the dispersion curve itself; the parameter chain producing those layer inputs -# is pinned separately by diiid_slayer_n1. Seconds to run — fast-tier eligible. +# is pinned separately by diiid_slayer_n1. Seconds to run — fast-tier eligible. No golden file yet. [case] name = "slayer_delta_probe" description = "SLAYER inner-layer Δ(Q) on a fixed 4×4 Q grid, DIII-D-like 2/1 layer parameters" @@ -16,6 +16,7 @@ extract = "all_real" label = "probe grid Re(Q)" noise_threshold = 0 order = 10 +class = "topological" # exact input, not a converged output [quantities.q_im] h5path = "slayer_probe/Q_im" @@ -24,6 +25,7 @@ extract = "all_real" label = "probe grid Im(Q)" noise_threshold = 0 order = 11 +class = "topological" # exact input, not a converged output [quantities.delta_re] h5path = "slayer_probe/Delta_re" diff --git a/regression-harness/golden/slayer_dels_fitzpatrick.toml b/regression-harness/golden/slayer_dels_fitzpatrick.toml index cd528b434..be3bc29a2 100644 --- a/regression-harness/golden/slayer_dels_fitzpatrick.toml +++ b/regression-harness/golden/slayer_dels_fitzpatrick.toml @@ -8,7 +8,7 @@ julia_version = "1.11.6" manifest_sha = "7e5c34ad1722fa6addf94f7387b972fa8ac17ff86166793819b158496d495132" nthreads = 8 os_arch = "arm64-apple-darwin24.0.0" -reason = "Regenerate on the branch after the second adversarial review: scan axes and tau now topological, runtime no longer pinned; values unchanged from the Fitzpatrick-validated state" +reason = "Pin del_s/d_beta over the Fitzpatrick figs 6.2/6.3 (Q_hat, P_hat) grid, with the scan axes and tau exact; values unchanged from the figure-validated state" [values.dels_db_im] atol = 0.0 From 2271d2580a8faf2666187c6f828c37c9787a717a Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 11:06:12 -0400 Subject: [PATCH 23/30] Regression - BUGFIX - Keep skipping checksum-typed values when building golden entries Co-Authored-By: Claude Opus 5.5 --- regression-harness/src/golden.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index c82df1962..4ba582e9a 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -589,7 +589,7 @@ function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector eq.value_type == "missing" && continue spec = get(spec_by_name, eq.name, nothing) spec === nothing && continue - is_pinnable(spec) || continue + (is_pinnable(spec) && eq.value_type != "checksum") || continue class = infer_class(spec) prior = existing === nothing ? nothing : get(existing, eq.name, nothing) if prior !== nothing && prior.value_type != eq.value_type From 25e94c2389d6a5808636e18a27a3b0033dd815c7 Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 15:40:35 -0400 Subject: [PATCH 24/30] Regression - BUGFIX - Close the journal-mode pragma query so later transactions can open PRAGMA journal_mode returns a row, so an unclosed query leaves a statement in progress on the connection until GC finalizes it; a transaction opened before then fails with "cannot open savepoint - SQL statements in progress". Co-Authored-By: Claude Opus 5.5 --- regression-harness/src/database.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/regression-harness/src/database.jl b/regression-harness/src/database.jl index c9a06b303..b174e3074 100644 --- a/regression-harness/src/database.jl +++ b/regression-harness/src/database.jl @@ -84,7 +84,8 @@ end function open_database(path::String)::SQLite.DB db = SQLite.DB(path) - DBInterface.execute(db, "PRAGMA journal_mode=WAL") + # journal_mode returns a row; close the query so no statement is left in progress to block a later transaction. + DBInterface.close!(DBInterface.execute(db, "PRAGMA journal_mode=WAL")) DBInterface.execute(db, "PRAGMA foreign_keys=ON") for stmt in split(SCHEMA_SQL, ";") s = strip(stmt) From 5684f6f7f5f3f1c52a2196b59a10001cd6cd1f3f Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 15:40:35 -0400 Subject: [PATCH 25/30] Regression - FEATURE! - Guard golden re-pins against exceeding moves, stray atol and untracked sources - --update-golden refuses a re-pin in which any gating quantity leaves its old tolerance unless --accept-exceeding is given; accepted moves are recorded in [meta.exceeded.] (previous_version, previous, deviation, previous_rtol, previous_atol) so the git diff shows them. - A gating entry's atol must be 0 unless its value holds an exact zero; enforced on load and save, and a carried atol is dropped once the value is nonzero. - --update-golden also refuses untracked files under examples/, src/ and regression-harness/cases/. Co-Authored-By: Claude Opus 5.5 --- docs/development/regression-harness.md | 4 +- regression-harness/regress.jl | 15 +++- regression-harness/src/golden.jl | 94 ++++++++++++++++++-- regression-harness/test/runtests.jl | 118 ++++++++++++++++++++++++- 4 files changed, 216 insertions(+), 15 deletions(-) diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md index 0f61d4773..32d1c2948 100644 --- a/docs/development/regression-harness.md +++ b/docs/development/regression-harness.md @@ -151,7 +151,9 @@ regress --update-golden --cases slayer_dels_fitzpatrick --reason "..." # re-pi - **Classes** come from the case file's `class` key (else `infer_class` in `src/golden.jl`), never from the golden file: `topological` (exact), `equilibrium_scalar`, `physics_converged` gate; `diagnostic` and `unconverged` are reported only. - **Tolerances** are either the provisional class default (`tolerance_basis = "class-default (provisional)"`) or `"measured"`, which requires a recorded `plateau_drift` or `platform_spread` no larger than `rtol`. No harness tool measures these yet, so every committed golden currently gates on provisional defaults, and `--check` labels those rows `provisional`. -- **Re-pinning** requires `--cases` and `--reason`, refuses a working tree with uncommitted tracked changes, and prints an old → new line per moved quantity, flagging any move the old tolerance would have failed. Such an entry loses its recorded evidence and becomes provisional, with a tolerance no looser than before. +- **`atol`** must be 0 on a gating entry unless its golden value contains an exact zero (a zero scalar, or an array element that is zero throughout); the relative test is the gate everywhere else. Loading and saving both enforce this. +- **Re-pinning** requires `--cases` and `--reason`, refuses a working tree with uncommitted tracked changes or with untracked files under `examples/`, `src/` or `regression-harness/cases/`, and prints an old → new line per moved quantity, flagging any move the old tolerance would have failed. +- **A move beyond the old tolerance** makes `--update-golden` refuse the whole re-pin unless `--accept-exceeding` is given. When it is, the entry loses its recorded evidence and becomes provisional, with a tolerance no looser than before, and the golden file's `[meta.exceeded.]` table records the previous version's value, the deviation and the old `rtol`/`atol`, so the reviewer sees each accepted move in the git diff. - A failing `--check` is fixed by explaining the physics or fixing the regression, never by widening `rtol` or `atol`. The unit tests for this logic run with `julia --project=regression-harness regression-harness/test/runtests.jl` and need no GPEC build. ## Exit status diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 1f097e7c4..7d9988684 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -134,7 +134,11 @@ Options: --check Run the working tree and compare against the committed golden values (regression-harness/golden/.toml). No CI workflow runs it yet. --update-golden Regenerate the golden values from a fresh run. Requires --cases and - --reason, and refuses a working tree with uncommitted changes. + --reason, and refuses a working tree with uncommitted changes or with + untracked files under examples/, src/ or regression-harness/cases/. + --accept-exceeding With --update-golden, allow a re-pin in which a gating quantity moved + outside its old tolerance (refused otherwise). Each such move's old + value and deviation is recorded under [meta.exceeded] in the golden file. --reason "..." Why the goldens changed. Mandatory with --update-golden: a golden change is a claim about physics that a reviewer has to evaluate. --help Print this help message @@ -189,7 +193,9 @@ Examples: """ function main(args=ARGS) - opts = parse_args(args) + # Kept out of CLIOptions so this flag needs no change to the shared option struct. + accept_exceeding = "--accept-exceeding" in args + opts = parse_args(filter(!=("--accept-exceeding"), args)) if opts.help print(HELP_TEXT) @@ -233,6 +239,9 @@ function main(args=ARGS) error("--update-golden requires --reason \"...\": a golden change is a claim about physics, " * "and the reviewer needs to know what changed and why") end + if accept_exceeding && !opts.update_golden + error("--accept-exceeding only applies to --update-golden") + end if opts.update_golden && isempty(opts.cases) error("--update-golden requires --cases: goldens are regenerated one deliberate case at a time") end @@ -297,7 +306,7 @@ function main(args=ARGS) if opts.update_golden update_golden_from_run(db, case_spec, resolved_refs[1].commit_hash, - String(opts.reason), REPO_ROOT) + String(opts.reason), REPO_ROOT; accept_exceeding=accept_exceeding) elseif opts.check summary = report_golden_check(db, case_spec, resolved_refs[1].commit_hash) n_golden_fail += summary.n_fail diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index 4ba582e9a..ec7b0c037 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -30,6 +30,10 @@ How strictly a quantity must reproduce, and why. A quantity's class comes from its case file (an explicit `class` key, else `infer_class`), never from the golden file, so demoting a gate to a non-gating class is a reviewed case-file change. + +A gating entry's `atol` must be 0 unless its golden value contains an exact zero (a scalar equal to +0, or an array element whose every component is 0). Everywhere else the relative test is the whole +gate; an absolute floor is only for structural zeros, where the relative test is meaningless. """ const TOLERANCE_CLASSES = ("topological", "equilibrium_scalar", "physics_converged", "diagnostic", "unconverged") @@ -89,6 +93,20 @@ end is_gating(g::GoldenValue) = g.class in GATING_CLASSES +_is_exact_zero(x) = x isa Number ? x == 0 : (x isa Vector && !isempty(x) && all(_is_exact_zero, x)) + +""" +True when a value holds an exact zero at the granularity `compare_to_golden` judges it: a zero +scalar, or an array element (a number, or a nested row or complex pair) that is zero throughout. +""" +function has_exact_zero(value_type, value_real, value_int, value_text) + value_type == "real" && return value_real !== nothing && value_real == 0.0 + value_type == "integer" && return value_int !== nothing && value_int == 0 + value_type == "json_array" && value_text !== nothing && return any(_is_exact_zero, JSON.parse(value_text; allownan=true)) + return false +end +has_exact_zero(g::GoldenValue) = has_exact_zero(g.value_type, g.value_real, g.value_int, g.value_text) + """ True when a tolerance is not backed by a recorded measurement, i.e. any basis other than "measured". """ @@ -108,6 +126,8 @@ function validate_golden_value(g::GoldenValue, context::AbstractString) if is_gating(g) (isfinite(g.rtol) && g.rtol >= 0 && isfinite(g.atol) && g.atol >= 0) || error("$context: gating quantity '$(g.name)' needs a finite, non-negative rtol and atol (got rtol=$(g.rtol), atol=$(g.atol))") + g.atol == 0 || has_exact_zero(g) || + error("$context: gating quantity '$(g.name)' has atol $(g.atol) but no exact zero in its value; atol is only for structural zeros") for (label, m) in (("platform_spread", g.platform_spread), ("plateau_drift", g.plateau_drift)) isfinite(m) && g.rtol < m && error( @@ -123,6 +143,13 @@ end """ Provenance for a whole golden file: what produced these numbers and why they last changed. + +## Fields + + - `exceeded` — for each gating quantity whose re-pin fell outside its previous tolerance (allowed + only with `--accept-exceeding`), a table of `previous_version`, `previous` (the old value), + `deviation` (as `compare_to_golden` reports it; NaN on a type change), `previous_rtol` and + `previous_atol`; written as `[meta.exceeded.]` so the git diff shows each accepted move """ struct GoldenMeta case::String @@ -135,8 +162,12 @@ struct GoldenMeta manifest_sha::String nthreads::Int blas_threads::Int + exceeded::Dict{String,Any} end +GoldenMeta(case, golden_version, generated_at, commit, reason, julia_version, os_arch, manifest_sha, nthreads, blas_threads) = + GoldenMeta(case, golden_version, generated_at, commit, reason, julia_version, os_arch, manifest_sha, nthreads, blas_threads, Dict{String,Any}()) + golden_path(case_name::AbstractString) = joinpath(GOLDEN_DIR, "$(case_name).toml") has_golden(case_name::AbstractString) = isfile(golden_path(case_name)) @@ -159,7 +190,8 @@ function load_golden(case_name::AbstractString) get(m, "os_arch", ""), get(m, "manifest_sha", ""), Int(get(m, "nthreads", -1)), - Int(get(m, "blas_threads", -1)) + Int(get(m, "blas_threads", -1)), + Dict{String,Any}(get(m, "exceeded", Dict{String,Any}())) ) values = Dict{String,GoldenValue}() for (name, v) in get(data, "values", Dict{String,Any}()) @@ -216,6 +248,7 @@ function save_golden(meta::GoldenMeta, values::Dict{String,GoldenValue}) "blas_threads" => meta.blas_threads ) ) + isempty(meta.exceeded) || (out["meta"]["exceeded"] = meta.exceeded) vals = Dict{String,Any}() for (name, g) in values entry = Dict{String,Any}( @@ -463,11 +496,39 @@ function report_golden_check(db::SQLite.DB, case_spec::CaseSpec, commit_hash::St end """ -Uncommitted changes to tracked files under `repo_root`, ignoring `exclude_dir` (the goldens themselves). +Directories, relative to the repository root, whose untracked files could change a golden run's result. """ -function uncommitted_changes(repo_root::AbstractString, exclude_dir::AbstractString) +const GOLDEN_SOURCE_DIRS = ("examples", "src", "regression-harness/cases") + +""" +Changes that the recorded commit would not reproduce: uncommitted changes to tracked files under +`repo_root`, ignoring `exclude_dir` (the goldens themselves), plus untracked files under `source_dirs`. +""" +function uncommitted_changes(repo_root::AbstractString, exclude_dir::AbstractString; source_dirs=GOLDEN_SOURCE_DIRS) rel = relpath(exclude_dir, repo_root) - return String(strip(read(`git -C $repo_root status --porcelain --untracked-files=no -- . ":(exclude)$rel"`, String))) + tracked = strip(read(`git -C $repo_root status --porcelain --untracked-files=no -- . ":(exclude)$rel"`, String)) + status = read(`git -C $repo_root status --porcelain --untracked-files=all -- $(collect(source_dirs))`, String) + untracked = filter(l -> startswith(l, "??"), split(status, '\n')) + return String(strip(join([tracked; untracked], '\n'))) +end + +""" +Gating entries of `existing` whose regenerated value in `values` falls outside the old tolerance, +as `name => Dict` tables ready for `GoldenMeta.exceeded`. A value-type change always counts. +""" +function exceeding_changes(existing::Dict{String,GoldenValue}, values::Dict{String,GoldenValue}, previous_version::Int) + exceeded = Dict{String,Any}() + for (name, new) in values + old = get(existing, name, nothing) + (old === nothing || !is_gating(old)) && continue + q = (value_real=new.value_real, value_int=new.value_int, value_text=new.value_text, value_type=new.value_type) + passed, dev, _ = compare_to_golden(q, old) + passed && continue + exceeded[name] = Dict{String,Any}("previous_version" => previous_version, + "previous" => something(old.value_real, old.value_int, old.value_text, ""), + "deviation" => dev, "previous_rtol" => old.rtol, "previous_atol" => old.atol) + end + return exceeded end """ @@ -476,17 +537,20 @@ Regenerate a case's golden file from a completed run, reporting what moved. Prints an old→new delta for every quantity whose value or class changed, flagging any move that the old tolerance would have failed, so the diff a reviewer sees in git comes with the size and significance of each move. Refuses a working tree with uncommitted changes outside the golden -directory, because the recorded commit is what a reviewer uses to reproduce a disputed number. +directory, or untracked files under `GOLDEN_SOURCE_DIRS`, because the recorded commit is what a +reviewer uses to reproduce a disputed number. Refuses a re-pin in which any gating quantity leaves +its old tolerance unless `accept_exceeding`, which records each such move in `[meta.exceeded]`. """ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash::String, - reason::String, repo_root::String) + reason::String, repo_root::String; accept_exceeding::Bool=false) info = get_run_info(db, commit_hash, case_spec.name) (info === nothing || !info.success) && error("Cannot update goldens for '$(case_spec.name)': the run did not succeed") if commit_hash == LOCAL_REF dirty = uncommitted_changes(repo_root, GOLDEN_DIR) isempty(dirty) || error( - "Cannot update goldens for '$(case_spec.name)': the working tree has uncommitted " * - "changes, so the recorded commit would not reproduce these values. Commit first.\n$dirty" + "Cannot update goldens for '$(case_spec.name)': the working tree has uncommitted changes " * + "or untracked source files, so the recorded commit would not reproduce these values. " * + "Commit (or remove) them first.\n$dirty" ) end @@ -504,12 +568,13 @@ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash: existing = previous === nothing ? nothing : previous.values values = build_golden_values(extracted, case_spec.quantities, existing) + exceeded = existing === nothing ? Dict{String,Any}() : exceeding_changes(existing, values, previous.meta.golden_version) fp = info.fingerprint meta = GoldenMeta(case_spec.name, previous === nothing ? 1 : previous.meta.golden_version + 1, Dates.format(Dates.now(), "yyyy-mm-dd"), strip(read(`git -C $repo_root rev-parse --short $(commit_hash == LOCAL_REF ? "HEAD" : commit_hash)`, String)), - reason, fp.julia_version, fp.os_arch, fp.manifest_sha, fp.nthreads, fp.blas_threads) + reason, fp.julia_version, fp.os_arch, fp.manifest_sha, fp.nthreads, fp.blas_threads, exceeded) println() println("Golden update: $(case_spec.name) (v$(previous === nothing ? 0 : previous.meta.golden_version) → v$(meta.golden_version))") @@ -536,6 +601,15 @@ function update_golden_from_run(db::SQLite.DB, case_spec::CaseSpec, commit_hash: line === nothing || println(@sprintf(" %-34s %s", name, line)) end end + if !isempty(exceeded) + names = join(sort(collect(keys(exceeded))), ", ") + accept_exceeding || error( + "Refusing to re-pin '$(case_spec.name)': $(length(exceeded)) gating quantity/quantities moved " * + "outside the old tolerance ($names). If --reason explains the move, re-run with --accept-exceeding; " * + "the old values and deviations are then recorded under [meta.exceeded]." + ) + println(" Accepted $(length(exceeded)) move(s) beyond the old tolerance, recorded under [meta.exceeded]: $names") + end path = save_golden(meta, values) println("Wrote $path ($(length(values)) quantities)") provisional = count(g -> is_gating(g) && is_provisional(g), Base.values(values)) @@ -613,6 +687,8 @@ function build_golden_values(extracted::Vector{ExtractedQuantity}, specs::Vector rtol, atol, basis = prior.rtol, prior.atol, prior.tolerance_basis drift, spread, at = prior.plateau_drift, prior.platform_spread, prior.converged_at end + # An absolute floor carried from a zero value is dropped once the value is no longer zero. + class in GATING_CLASSES && !has_exact_zero(eq.value_type, eq.value_real, eq.value_int, eq.value_text) && (atol = 0.0) values[eq.name] = GoldenValue(eq.name, eq.value_type, eq.value_real, eq.value_int, eq.value_text, rtol, atol, class, basis, drift, spread, at) end diff --git a/regression-harness/test/runtests.jl b/regression-harness/test/runtests.jl index a36ca5817..e28f8bd53 100644 --- a/regression-harness/test/runtests.jl +++ b/regression-harness/test/runtests.jl @@ -163,7 +163,8 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val basis="measured", drift=1e-8, spread=2e-7, at="mpsi=1024")) vals2 = build_golden_values([extracted_q("x"; value_real=1.0 + 1e-7)], [spec], prior2) @test vals2["x"].value_real == 1.0 + 1e-7 - @test vals2["x"].rtol == 3e-7 && vals2["x"].atol == 1e-12 + # An absolute floor on a nonzero value is not a legal gate, so it is dropped rather than carried. + @test vals2["x"].rtol == 3e-7 && vals2["x"].atol == 0.0 @test vals2["x"].tolerance_basis == "measured" @test vals2["x"].plateau_drift == 1e-8 && vals2["x"].platform_spread == 2e-7 @@ -174,6 +175,11 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val @test vals5["x"].rtol == 3e-7 && vals5["x"].atol == 0.0 @test vals5["x"].tolerance_basis == "provisional" && is_provisional(vals5["x"]) @test isnan(vals5["x"].plateau_drift) && isnan(vals5["x"].platform_spread) && isempty(vals5["x"].converged_at) + + # A structural zero keeps its absolute floor across a re-pin that stays zero. + priorz = Dict("z" => golden_val("z"; value_type="json_array", value_text="[0.0,1.0]", atol=1e-10)) + valsz = build_golden_values([extracted_q("z"; value_text="[0.0,1.0000001]", value_type="json_array")], [qspec("z"; type="real_array")], priorz) + @test valsz["z"].atol == 1e-10 # A measured tolerance wider than the class default is tightened back to the default. prior5 = Dict("x" => golden_val("x"; value_real=1.0, rtol=1e-4, basis="measured", drift=5e-5)) vals6 = @test_logs (:warn, r"beyond its old tolerance") build_golden_values( @@ -263,6 +269,49 @@ value = 1.0 ) @test_throws ErrorException load_golden("below_drift") + # atol on a gating entry is only for structural zeros. + write( + golden_path("atol_nonzero"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-6 +atol = 1.0e-12 +value = 1.0 +""" + ) + @test_throws ErrorException load_golden("atol_nonzero") + write( + golden_path("atol_zeros"), + """ +[values.x] +class = "physics_converged" +rtol = 1.0e-6 +atol = 1.0e-12 +value = 0.0 + +[values.v] +class = "physics_converged" +value_type = "json_array" +value_text = "[1.0,0.0]" +rtol = 1.0e-6 +atol = 1.0e-12 + +[values.c] +class = "physics_converged" +value_type = "json_array" +value_text = "[[1.0,0.0],[0.0,0.0]]" +rtol = 1.0e-6 +atol = 1.0e-12 +""" + ) + @test length(load_golden("atol_zeros").values) == 3 + # A complex pair with only one zero component is not a structural zero. + @test !has_exact_zero(golden_val("c"; value_type="json_array", value_text="[[1.0,0.0],[2.0,0.5]]")) + @test has_exact_zero(golden_val("n"; value_type="integer", value_int=0)) + @test_throws ErrorException save_golden(GoldenMeta("atol_save", 1, "", "", "", "", "", "", 1, 1), + Dict("x" => golden_val("x"; value_real=2.0, atol=1e-9))) + # "measured" must be backed by a recorded measurement. write( golden_path("unbacked"), @@ -315,6 +364,63 @@ value = 12.5 @test back.values["a"].plateau_drift == 1e-8 && back.values["a"].platform_spread == 1e-7 @test back.values["n"].value_int == 7 && back.values["n"].class == "topological" @test back.values["v"].value_text == "[1.0,2.0]" + @test isempty(back.meta.exceeded) + + # Accepted moves beyond the old tolerance round-trip through [meta.exceeded]. + exc = exceeding_changes(Dict("a" => golden_val("a"; value_real=2.5)), Dict("a" => golden_val("a"; value_real=2.6)), 1) + meta2 = GoldenMeta("rt_case", 2, "2026-09-25", "deadbeef", "unit test", "1.11.6", "arm64", "abc", 4, 4, exc) + save_golden(meta2, Dict("a" => golden_val("a"; value_real=2.6))) + back2 = load_golden("rt_case") + @test back2.meta.exceeded["a"]["previous"] == 2.5 && back2.meta.exceeded["a"]["previous_version"] == 1 + @test back2.meta.exceeded["a"]["deviation"] ≈ 0.04 + @test occursin("[meta.exceeded.a]", read(golden_path("rt_case"), String)) + end + + @testset "exceeding_changes" begin + existing = Dict( + "in" => golden_val("in"; value_real=1.0), + "out" => golden_val("out"; value_real=1.0), + "diag" => golden_val("diag"; value_real=1.0, rtol=Inf, atol=Inf, class="diagnostic"), + "typ" => golden_val("typ"; value_real=1.0), + "tok" => golden_val("tok"; value_type="token", value_text="riccati", class="topological")) + values = Dict( + "in" => golden_val("in"; value_real=1.0 + 1e-7), + "out" => golden_val("out"; value_real=1.1), + "diag" => golden_val("diag"; value_real=50.0, rtol=Inf, atol=Inf, class="diagnostic"), + "typ" => golden_val("typ"; value_type="json_array", value_text="[1.0]"), + "tok" => golden_val("tok"; value_type="token", value_text="galerkin", class="topological"), + "new" => golden_val("new"; value_real=3.0)) + exc = exceeding_changes(existing, values, 4) + @test sort(collect(keys(exc))) == ["out", "tok", "typ"] + @test exc["out"]["previous"] == 1.0 && exc["out"]["deviation"] ≈ 0.1 && exc["out"]["previous_version"] == 4 + @test exc["tok"]["previous"] == "riccati" + @test isnan(exc["typ"]["deviation"]) + end + + @testset "update refuses a move beyond the old tolerance" begin + repo = mktempdir() + run(`git -C $repo init -q`) + write(joinpath(repo, "f.txt"), "x\n") + run(`git -C $repo add -A`) + run(`git -C $repo -c user.name=t -c user.email=t@t commit -q -m init`) + sha = String(strip(read(`git -C $repo rev-parse HEAD`, String))) + db = open_database(joinpath(mktempdir(), "update.sqlite")) + case = CaseSpec("exceed_case", "synthetic", "", [qspec("a"), qspec("b")], "example", Dict{String,Any}()) + save_golden(GoldenMeta("exceed_case", 3, "2026-09-01", "cafe", "seed", "1.11.6", "arm64", "abc", 4, 4), + Dict("a" => golden_val("a"; value_real=1.0), "b" => golden_val("b"; value_real=2.0))) + store_run(db, sha, sha, "", "", "exceed_case", 1.0, [extracted_q("a"; value_real=1.2), extracted_q("b"; value_real=2.0)]) + @test_throws ErrorException redirect_stdout(devnull) do + update_golden_from_run(db, case, sha, "moved", repo) + end + @test load_golden("exceed_case").meta.golden_version == 3 + @test_logs (:warn, r"beyond its old tolerance") match_mode=:any redirect_stdout(devnull) do + update_golden_from_run(db, case, sha, "moved", repo; accept_exceeding=true) + end + back = load_golden("exceed_case") + @test back.meta.golden_version == 4 && back.values["a"].value_real == 1.2 + @test collect(keys(back.meta.exceeded)) == ["a"] + @test back.meta.exceeded["a"]["previous"] == 1.0 && back.meta.exceeded["a"]["previous_version"] == 3 + close_database(db) end @testset "infer_class" begin @@ -433,10 +539,18 @@ value = 12.5 run(`git -C $repo add -A`) run(`git -C $repo -c user.name=t -c user.email=t@t commit -q -m init`) @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) - # Rewriting a golden file is the update itself, and untracked files are not source. + # Rewriting a golden file is the update itself, and untracked files outside the source dirs are not source. write(joinpath(repo, "golden", "case.toml"), "a = 2\n") write(joinpath(repo, "scratch.md"), "notes\n") @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) + # An untracked file under a source dir can change the run, so it is refused like a tracked edit. + for f in ("src/new.jl", "examples/case/gpec.toml", "regression-harness/cases/new_case.toml") + mkpath(dirname(joinpath(repo, f))) + write(joinpath(repo, f), "x\n") + @test occursin(f, uncommitted_changes(repo, joinpath(repo, "golden"))) + rm(joinpath(repo, f)) + end + @test isempty(uncommitted_changes(repo, joinpath(repo, "golden"))) write(joinpath(repo, "src.jl"), "x = 2\n") @test occursin("src.jl", uncommitted_changes(repo, joinpath(repo, "golden"))) end From 3d85ed77636afe1e9e2cee702434c743089b9e03 Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 15:38:32 -0400 Subject: [PATCH 26/30] Regression - API! - Require an explicit tolerance class on every regression quantity Every [quantities.*] block in a case file now declares its golden tolerance class, and infer_class no longer guesses one from the quantity's name or type, so renaming a quantity cannot silently change whether it gates. Each class written into the case files is the one infer_class returned before this change, so no quantity changes class; load_case errors on a missing or unknown class, naming the file and quantity. Co-Authored-By: Claude Opus 5.5 --- docs/development/regression-harness.md | 2 +- docs/development/toml-conventions.md | 3 +- regression-harness/cases/diiid_n1.toml | 49 +++++++++++++++++++ .../cases/diiid_n1_riccati.toml | 19 +++++++ regression-harness/cases/diiid_slayer_n1.toml | 24 +++++++++ .../cases/efit_fixedbdy_separatrix.toml | 6 +++ .../cases/gal_resistive_diiid.toml | 12 +++++ .../cases/gal_resistive_pe.toml | 10 ++++ regression-harness/cases/ggj_ray_q500i.toml | 5 ++ regression-harness/cases/ggj_reference.toml | 5 ++ .../cases/slayer_dels_fitzpatrick.toml | 3 ++ .../cases/slayer_delta_probe.toml | 3 ++ .../cases/solovev_kinetic_calculated.toml | 15 ++++++ .../cases/solovev_kinetic_multiion.toml | 7 +++ .../cases/solovev_kinetic_ntv.toml | 6 +++ .../cases/solovev_kinetic_nuzero.toml | 15 ++++++ regression-harness/cases/solovev_multi_n.toml | 16 ++++++ regression-harness/cases/solovev_n1.toml | 23 +++++++++ regression-harness/src/config.jl | 5 +- regression-harness/src/golden.jl | 32 ++++-------- regression-harness/src/types.jl | 5 +- regression-harness/test/runtests.jl | 44 +++++++++++------ 22 files changed, 266 insertions(+), 43 deletions(-) diff --git a/docs/development/regression-harness.md b/docs/development/regression-harness.md index 32d1c2948..60cb889ad 100644 --- a/docs/development/regression-harness.md +++ b/docs/development/regression-harness.md @@ -149,7 +149,7 @@ regress --check --cases slayer_dels_fitzpatrick # worki regress --update-golden --cases slayer_dels_fitzpatrick --reason "..." # re-pin from a fresh run ``` -- **Classes** come from the case file's `class` key (else `infer_class` in `src/golden.jl`), never from the golden file: `topological` (exact), `equilibrium_scalar`, `physics_converged` gate; `diagnostic` and `unconverged` are reported only. +- **Classes** come only from the `class` key that every `[quantities.*]` block in a case file must declare, never from the golden file or the quantity's name; a missing or unknown class makes the case fail to load. `topological` (exact), `equilibrium_scalar`, `physics_converged` gate; `diagnostic` and `unconverged` are reported only. - **Tolerances** are either the provisional class default (`tolerance_basis = "class-default (provisional)"`) or `"measured"`, which requires a recorded `plateau_drift` or `platform_spread` no larger than `rtol`. No harness tool measures these yet, so every committed golden currently gates on provisional defaults, and `--check` labels those rows `provisional`. - **`atol`** must be 0 on a gating entry unless its golden value contains an exact zero (a zero scalar, or an array element that is zero throughout); the relative test is the gate everywhere else. Loading and saving both enforce this. - **Re-pinning** requires `--cases` and `--reason`, refuses a working tree with uncommitted tracked changes or with untracked files under `examples/`, `src/` or `regression-harness/cases/`, and prints an old → new line per moved quantity, flagging any move the old tolerance would have failed. diff --git a/docs/development/toml-conventions.md b/docs/development/toml-conventions.md index 5ee2c3d5d..2949dbded 100644 --- a/docs/development/toml-conventions.md +++ b/docs/development/toml-conventions.md @@ -16,7 +16,8 @@ modify them. **Do not invent a new convention** — match what the existing exam files** — `examples/DIIID-like_ideal_example/gpec.toml` is the canonical reference for the common sections. Exception: in `regression-harness/cases/*.toml` the repeated `[quantities.*]` schema keys (`h5path`, `type`, `extract`, `label`, `noise_threshold`, - `order`) are developer metadata and do **not** need inline comments — a case file needs + `order`, and the required golden tolerance `class`) are developer metadata and do **not** + need inline comments (a `class` that is not the obvious one for its quantity should say why) — a case file needs the header plus an informative block comment per quantity group instead. 3. **Section comments only when informative.** No comment is required above a section. Keep a block comment only when it carries real information (e.g. the Solovev `[Wall]` note on why a diff --git a/regression-harness/cases/diiid_n1.toml b/regression-harness/cases/diiid_n1.toml index dcba09788..e859c98bd 100644 --- a/regression-harness/cases/diiid_n1.toml +++ b/regression-harness/cases/diiid_n1.toml @@ -18,6 +18,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -26,6 +27,7 @@ extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -34,6 +36,7 @@ extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 order = 12 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -42,6 +45,7 @@ extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.vacuum_eigenvalue] h5path = "ForceFreeStates/FreeBoundaryStability/vacuum_eigenvalue" @@ -50,6 +54,7 @@ extract = "value" label = "vacuum matrix min eigenvalue" noise_threshold = 1e-10 order = 14 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.ep_all] @@ -59,6 +64,7 @@ extract = "all_complex" label = "plasma energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ev_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -67,6 +73,7 @@ extract = "all_complex" label = "vacuum energy (all)" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -75,6 +82,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 22 +class = "physics_converged" # Integration [quantities.nstep] @@ -84,6 +92,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -92,6 +101,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 31 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -101,6 +111,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 40 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -109,6 +120,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 41 +class = "equilibrium_scalar" [quantities.betat] h5path = "Equilibrium/beta_t" @@ -117,6 +129,7 @@ extract = "value" label = "beta_t" noise_threshold = 1e-12 order = 42 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -125,6 +138,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 43 +class = "equilibrium_scalar" [quantities.li1] h5path = "Equilibrium/l_i_1" @@ -133,6 +147,7 @@ extract = "value" label = "internal inductance li1" noise_threshold = 1e-12 order = 44 +class = "equilibrium_scalar" [quantities.li2] h5path = "Equilibrium/l_i_2" @@ -141,6 +156,7 @@ extract = "value" label = "internal inductance li2" noise_threshold = 1e-12 order = 45 +class = "equilibrium_scalar" [quantities.li3] h5path = "Equilibrium/l_i_3" @@ -149,6 +165,7 @@ extract = "value" label = "internal inductance li3" noise_threshold = 1e-12 order = 46 +class = "equilibrium_scalar" [quantities.betap1] h5path = "Equilibrium/beta_p_1" @@ -157,6 +174,7 @@ extract = "value" label = "poloidal beta betap1" noise_threshold = 1e-12 order = 47 +class = "equilibrium_scalar" [quantities.betap2] h5path = "Equilibrium/beta_p_2" @@ -165,6 +183,7 @@ extract = "value" label = "poloidal beta betap2" noise_threshold = 1e-12 order = 48 +class = "equilibrium_scalar" [quantities.betap3] h5path = "Equilibrium/beta_p_3" @@ -173,6 +192,7 @@ extract = "value" label = "poloidal beta betap3" noise_threshold = 1e-12 order = 49 +class = "equilibrium_scalar" [quantities.betaj] h5path = "Equilibrium/beta_j" @@ -181,6 +201,7 @@ extract = "value" label = "current beta betaj" noise_threshold = 1e-12 order = 57 +class = "equilibrium_scalar" [quantities.volume] h5path = "Equilibrium/volume" @@ -189,6 +210,7 @@ extract = "value" label = "plasma volume" noise_threshold = 1e-12 order = 58 +class = "equilibrium_scalar" [quantities.crnt] h5path = "Equilibrium/I_p" @@ -197,6 +219,7 @@ extract = "value" label = "plasma current" noise_threshold = 1e-12 order = 59 +class = "equilibrium_scalar" [quantities.bt0] h5path = "Equilibrium/B_T_axis" @@ -205,6 +228,7 @@ extract = "value" label = "toroidal field bt0" noise_threshold = 1e-12 order = 62 +class = "equilibrium_scalar" [quantities.bwall] h5path = "Equilibrium/B_T_wall" @@ -213,6 +237,7 @@ extract = "value" label = "wall field bwall" noise_threshold = 1e-12 order = 63 +class = "equilibrium_scalar" [quantities.aratio] h5path = "Equilibrium/aspect_ratio" @@ -221,6 +246,7 @@ extract = "value" label = "aspect ratio" noise_threshold = 1e-12 order = 64 +class = "equilibrium_scalar" [quantities.kappa] h5path = "Equilibrium/kappa" @@ -229,6 +255,7 @@ extract = "value" label = "elongation kappa" noise_threshold = 1e-12 order = 65 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -238,6 +265,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 50 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -246,6 +274,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 51 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -254,6 +283,7 @@ extract = "all_real" label = "singular q values" noise_threshold = 1e-8 order = 52 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -263,6 +293,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 60 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -271,6 +302,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 61 +class = "topological" [quantities.island_half_width] h5path = "PerturbedEquilibrium/SingularCoupling/island_half_width" @@ -279,6 +311,7 @@ extract = "all_real" label = "island half-widths" noise_threshold = 1e-8 order = 81 +class = "physics_converged" [quantities.chirikov_parameter] h5path = "PerturbedEquilibrium/SingularCoupling/chirikov_parameter" @@ -287,6 +320,7 @@ extract = "all_real" label = "Chirikov parameter" noise_threshold = 1e-8 order = 82 +class = "physics_converged" [quantities.resonant_area_weighted_field] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -295,6 +329,7 @@ extract = "norm" label = "||resonant area-weighted field||" noise_threshold = 1e-8 order = 83 +class = "physics_converged" [quantities.resonant_area_weighted_field_all] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -302,6 +337,7 @@ type = "complex_vector" extract = "all_complex" label = "resonant area-weighted field b^r" noise_threshold = 1e-8 +class = "physics_converged" # Perturbed equilibrium: energies [quantities.pe_plasma_energy] @@ -311,6 +347,7 @@ extract = "value" label = "PE plasma energy" noise_threshold = 1e-10 order = 91 +class = "physics_converged" [quantities.pe_vacuum_energy] h5path = "PerturbedEquilibrium/Energies/vacuum_energy" @@ -319,6 +356,7 @@ extract = "value" label = "PE vacuum energy" noise_threshold = 1e-10 order = 92 +class = "physics_converged" [quantities.pe_surface_energy] h5path = "PerturbedEquilibrium/Energies/surface_energy" @@ -327,6 +365,7 @@ extract = "value" label = "PE surface energy" noise_threshold = 1e-10 order = 93 +class = "physics_converged" [quantities.pe_toroidal_torque] h5path = "PerturbedEquilibrium/Energies/toroidal_torque" @@ -335,6 +374,7 @@ extract = "value" label = "PE toroidal torque" noise_threshold = 1e-12 order = 94 +class = "physics_converged" # NTV (neoclassical toroidal viscosity) toroidal torque T_φ [N·m] from the # [KineticForces] FGAR method (pressure moment) — a diagnostic computed on top of @@ -348,6 +388,7 @@ extract = "value" label = "NTV torque FGAR [N·m]" noise_threshold = 1e-2 order = 95 +class = "physics_converged" # NTV kinetic energy δW_k = Im(T)/(2n) [J] through the same post-PE FGAR path. # Tracked at a tighter threshold — it isolates the energy component the @@ -359,6 +400,7 @@ extract = "value" label = "NTV kinetic energy dW FGAR [J]" noise_threshold = 1e-3 order = 96 +class = "physics_converged" # Profiles (checksums) [quantities.q_profile] @@ -368,6 +410,7 @@ extract = "checksum" label = "q profile (checksum)" noise_threshold = 0 order = 70 +class = "physics_converged" [quantities.pressure_profile] h5path = "Equilibrium/Profiles/mu0p" @@ -376,6 +419,7 @@ extract = "checksum" label = "pressure profile (checksum)" noise_threshold = 0 order = 71 +class = "physics_converged" # Local stability profiles (local_stability_flag = true). Checksums hash the full # profile so they are robust to NaN edge-surface entries and catch any change. @@ -386,6 +430,7 @@ extract = "checksum" label = "Mercier D_I profile (checksum)" noise_threshold = 0 order = 72 +class = "physics_converged" [quantities.locstab_dr] h5path = "LocalStability/D_R" @@ -394,6 +439,7 @@ extract = "checksum" label = "resistive interchange D_R profile (checksum)" noise_threshold = 0 order = 73 +class = "physics_converged" [quantities.locstab_ballooning_dp] h5path = "LocalStability/ballooning_Delta_prime" @@ -402,6 +448,7 @@ extract = "checksum" label = "ballooning Delta' profile (checksum)" noise_threshold = 0 order = 74 +class = "physics_converged" # Runtime [quantities.runtime] @@ -411,6 +458,7 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" # Root-area-weighted (coordinate-invariant) forcing field b̃ on the control surface: the # applied coil spectrum pushed through the √(J|∇ψ|) weight operator. Tracks the field-space @@ -422,3 +470,4 @@ extract = "norm" label = "||forcing b~|| (root-area-weighted)" noise_threshold = 1e-10 order = 1000 +class = "physics_converged" diff --git a/regression-harness/cases/diiid_n1_riccati.toml b/regression-harness/cases/diiid_n1_riccati.toml index 811ed13c0..9b8442317 100644 --- a/regression-harness/cases/diiid_n1_riccati.toml +++ b/regression-harness/cases/diiid_n1_riccati.toml @@ -21,6 +21,7 @@ extract = "toml_key:ForceFreeStates.integrator" label = "integrator (Δ′ provenance)" noise_threshold = 0 order = 1 +class = "topological" # Tearing stability Δ' — the msing×msing BVP matrix diagonal, and the raw side-major # outer-region matrix it is projected from. @@ -31,6 +32,7 @@ extract = "diagonal_complex" label = "Δ′ STRIDE-like BVP (PEST3) diagonal" noise_threshold = 1e-8 order = 10 +class = "physics_converged" [quantities.delta_prime_raw] h5path = "SingularSurfaces/Delta_prime_raw" @@ -39,6 +41,7 @@ extract = "all_complex" label = "Δ′ STRIDE-like BVP (PEST3) raw side-major" noise_threshold = 1e-8 order = 11 +class = "physics_converged" [quantities.delta_coil] h5path = "SingularSurfaces/Delta_coil" @@ -47,6 +50,7 @@ extract = "all_complex" label = "edge coil response delta_coil" noise_threshold = 1e-8 order = 12 +class = "physics_converged" # Energies — leading generalized (W,N) pencil eigenvalues at the final truncation (psilim): power-normalized (⟨|ξ|²⟩ = 1 metric) and invariant to the working-coordinate Jacobian. [quantities.et_real] @@ -56,6 +60,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -64,6 +69,7 @@ extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -72,6 +78,7 @@ extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 order = 22 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -80,6 +87,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 23 +class = "physics_converged" # Singular surfaces — the Δ' matrix is indexed by these, so a shift here explains a Δ' shift. [quantities.msing] @@ -89,6 +97,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 30 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -97,6 +106,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 31 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -105,6 +115,7 @@ extract = "all_real" label = "singular q values" noise_threshold = 1e-8 order = 32 +class = "physics_converged" # Integration — chunk-endpoint step counts move when the chunk sizing changes. [quantities.nstep] @@ -114,6 +125,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 40 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -122,6 +134,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 41 +class = "diagnostic" # Mode numbers [quantities.mpert] @@ -131,6 +144,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 50 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -139,6 +153,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 51 +class = "topological" # Equilibrium — shared with diiid_n1; confirms the two cases sit on the same equilibrium. [quantities.q0] @@ -148,6 +163,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 60 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -156,6 +172,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 61 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -164,6 +181,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 62 +class = "equilibrium_scalar" # Runtime [quantities.runtime] @@ -173,3 +191,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/diiid_slayer_n1.toml b/regression-harness/cases/diiid_slayer_n1.toml index f969fe8cf..9824a728c 100644 --- a/regression-harness/cases/diiid_slayer_n1.toml +++ b/regression-harness/cases/diiid_slayer_n1.toml @@ -20,6 +20,7 @@ extract = "toml_key:ForceFreeStates.integrator" label = "integrator (Δ′ provenance)" noise_threshold = 0 order = 1 +class = "topological" # Per-surface SLAYER layer parameters (geometry + dimensionless) [quantities.slayer_ising] @@ -29,6 +30,7 @@ extract = "all_real" label = "SLAYER surface indices" noise_threshold = 0 order = 10 +class = "physics_converged" [quantities.slayer_m] h5path = "Tearing/PerSurface/m" @@ -37,6 +39,7 @@ extract = "all_real" label = "SLAYER poloidal m" noise_threshold = 0 order = 11 +class = "physics_converged" [quantities.slayer_n] h5path = "Tearing/PerSurface/n" @@ -45,6 +48,7 @@ extract = "all_real" label = "SLAYER toroidal n" noise_threshold = 0 order = 12 +class = "physics_converged" [quantities.slayer_rs] h5path = "Tearing/PerSurface/rs" @@ -53,6 +57,7 @@ extract = "all_real" label = "SLAYER minor radius rs" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.slayer_sval_r] h5path = "Tearing/PerSurface/sval_r" @@ -61,6 +66,7 @@ extract = "all_real" label = "SLAYER r-based shear" noise_threshold = 1e-10 order = 14 +class = "physics_converged" [quantities.slayer_lu] h5path = "Tearing/PerSurface/lu" @@ -69,6 +75,7 @@ extract = "all_real" label = "SLAYER Lundquist S" noise_threshold = 1e-8 order = 15 +class = "physics_converged" [quantities.slayer_D_norm] h5path = "Tearing/PerSurface/D_norm" @@ -77,6 +84,7 @@ extract = "all_real" label = "SLAYER D_norm" noise_threshold = 1e-10 order = 16 +class = "physics_converged" [quantities.slayer_P_perp] h5path = "Tearing/PerSurface/P_perp" @@ -85,6 +93,7 @@ extract = "all_real" label = "SLAYER P_perp" noise_threshold = 1e-8 order = 17 +class = "physics_converged" [quantities.slayer_tauk] h5path = "Tearing/PerSurface/tau_k" @@ -93,6 +102,7 @@ extract = "all_real" label = "SLAYER tauk" noise_threshold = 1e-12 order = 18 +class = "physics_converged" [quantities.slayer_iota_e] h5path = "Tearing/PerSurface/iota_e" @@ -101,6 +111,7 @@ extract = "all_real" label = "SLAYER iota_e" noise_threshold = 1e-12 order = 19 +class = "physics_converged" # Normalized diamagnetic frequencies. iota_e above is their ratio, so it is blind # to a common factor on both; these pin the drive itself. @@ -111,6 +122,7 @@ extract = "all_real" label = "SLAYER Q_e" noise_threshold = 1e-12 order = 20 +class = "physics_converged" [quantities.slayer_Q_i] h5path = "Tearing/PerSurface/Q_i" @@ -119,6 +131,7 @@ extract = "all_real" label = "SLAYER Q_i" noise_threshold = 1e-12 order = 21 +class = "physics_converged" # Tearing eigenvalue (coupled mode → length 1). The headline deliverable: # a real, nonzero growth rate on a realistic equilibrium. Root-extraction @@ -136,6 +149,7 @@ extract = "first_3_complex" label = "SLAYER Q_root [2/1,3/1,4/1]" noise_threshold = 1e-4 order = 30 +class = "physics_converged" [quantities.slayer_omega_Hz] h5path = "Tearing/Roots/omega" @@ -144,6 +158,7 @@ extract = "first_3" label = "SLAYER ω_Hz [2/1,3/1,4/1]" noise_threshold = 1.0 order = 32 +class = "physics_converged" [quantities.slayer_gamma_Hz] h5path = "Tearing/Roots/gamma" @@ -152,6 +167,7 @@ extract = "first_3" label = "SLAYER γ_Hz [2/1,3/1,4/1]" noise_threshold = 1e-1 order = 33 +class = "physics_converged" # no_root flag (1 = extraction failed). Pinned for the inner three surfaces; # this case guards that the 2/1, 3/1, 4/1 keep finding a real root. @@ -162,6 +178,7 @@ extract = "first_3" label = "SLAYER no_root flags [2/1,3/1,4/1]" noise_threshold = 0 order = 34 +class = "physics_converged" # Outer-region Δ' consumed by the SLAYER matching — the same STRIDE-like BVP (PEST3) matrix # the riccati case pins at source; tracked here too so a change in what SLAYER actually @@ -173,6 +190,7 @@ extract = "diagonal_complex" label = "Δ′ STRIDE-like BVP (PEST3) diagonal, as ingested by SLAYER" noise_threshold = 1e-8 order = 40 +class = "physics_converged" # Inner-layer solve outputs per surface (the resistive-layer Riccati solution itself, not the # outer matching). The adaptive-scan Δ(Q) sample arrays are deliberately NOT pinned: AMR sample @@ -184,6 +202,7 @@ extract = "all_complex" label = "SLAYER inner-layer thickness δ_s (complex, Riccati)" noise_threshold = 1e-10 order = 41 +class = "physics_converged" [quantities.slayer_delta_s_over_d_beta] h5path = "Tearing/LayerWidths/delta_s_over_d_beta" @@ -192,6 +211,7 @@ extract = "all_complex" label = "SLAYER inner-layer δ_s/d_β (complex, dimensionless)" noise_threshold = 1e-10 order = 42 +class = "physics_converged" [quantities.slayer_delta_s_abs] h5path = "Tearing/LayerWidths/delta_s_abs" @@ -200,6 +220,7 @@ extract = "all_real" label = "SLAYER inner-layer physical thickness |δ_s| [m]" noise_threshold = 1e-12 order = 43 +class = "physics_converged" [quantities.slayer_d_beta] h5path = "Tearing/LayerWidths/d_beta" @@ -208,6 +229,7 @@ extract = "all_real" label = "SLAYER β-weighted ion drift scale d_β [m]" noise_threshold = 1e-12 order = 44 +class = "physics_converged" # Settings (catches accidental config drift) [quantities.slayer_enabled] @@ -217,6 +239,7 @@ extract = "value" label = "SLAYER enabled flag" noise_threshold = 0 order = 90 +class = "topological" [quantities.runtime] h5path = "" @@ -225,3 +248,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/efit_fixedbdy_separatrix.toml b/regression-harness/cases/efit_fixedbdy_separatrix.toml index d64f09ba7..323150732 100644 --- a/regression-harness/cases/efit_fixedbdy_separatrix.toml +++ b/regression-harness/cases/efit_fixedbdy_separatrix.toml @@ -19,6 +19,7 @@ extract = "value" label = "psio (total poloidal flux)" noise_threshold = 1e-12 order = 10 +class = "equilibrium_scalar" [quantities.q0] h5path = "Equilibrium/q_axis" @@ -27,6 +28,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 11 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -35,6 +37,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 12 +class = "equilibrium_scalar" [quantities.betat] h5path = "Equilibrium/beta_t" @@ -43,6 +46,7 @@ extract = "value" label = "beta_t" noise_threshold = 1e-12 order = 13 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -51,6 +55,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 14 +class = "equilibrium_scalar" [quantities.runtime] h5path = "" @@ -59,3 +64,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/gal_resistive_diiid.toml b/regression-harness/cases/gal_resistive_diiid.toml index bc0079848..6af81496a 100644 --- a/regression-harness/cases/gal_resistive_diiid.toml +++ b/regression-harness/cases/gal_resistive_diiid.toml @@ -20,6 +20,7 @@ extract = "toml_key:ForceFreeStates.integrator" label = "integrator (Δ′ provenance)" noise_threshold = 0 order = 1 +class = "topological" # Number of resonant surfaces in the gal domain [quantities.gal_msing] @@ -29,6 +30,7 @@ extract = "value" label = "gal # singular surfaces" noise_threshold = 0 order = 10 +class = "topological" [quantities.gal_sing_q] h5path = "ForceFreeStates/Solutions/GalerkinIntegration/rational_q" @@ -37,6 +39,7 @@ extract = "all_real" label = "gal singular q values" noise_threshold = 1e-8 order = 11 +class = "physics_converged" # PEST-3 Δ matching matrix — per-surface diagonal (the physics-meaningful tearing Δ′) [quantities.gal_pest3_delta_diag] @@ -46,6 +49,7 @@ extract = "diagonal_complex" label = "Δ′ Galerkin RDCON (PEST3) diagonal" noise_threshold = 1e-6 order = 20 +class = "physics_converged" # Raw outer Δ′ matrix (2·msing × 2·msing) — Frobenius norm catches any element drift [quantities.gal_delta_norm] @@ -55,6 +59,7 @@ extract = "norm" label = "||Δ′ Galerkin RDCON (PEST3) matrix||" noise_threshold = 1e-6 order = 21 +class = "physics_converged" # rpec coil-response block (edge mode × surface-side) — Frobenius norm [quantities.gal_delta_coil_norm] @@ -64,6 +69,7 @@ extract = "norm" label = "||gal Δ_coil block||" noise_threshold = 1e-6 order = 22 +class = "physics_converged" # Mercier index per surface [quantities.gal_di] @@ -73,6 +79,7 @@ extract = "all_real" label = "gal D_I per surface" noise_threshold = 1e-8 order = 30 +class = "physics_converged" # Resonant exponents α per surface [quantities.gal_alpha] @@ -82,6 +89,7 @@ extract = "all_complex" label = "gal α per surface" noise_threshold = 1e-8 order = 31 +class = "physics_converged" # --- DRIVEN (RPEC) inner-layer matching (gal_match_flag) --- # Only finite, well-defined matching quantities are tracked. The reconstructed ξ/ξ′ are deliberately @@ -97,6 +105,7 @@ extract = "norm" label = "||gal inner-layer Δ||" noise_threshold = 1e-6 order = 41 +class = "physics_converged" # Outer-region matched coefficients cout (2·msing × mcoil) — the 4·msing matching assembly + solve. [quantities.gal_match_cout_norm] @@ -106,6 +115,7 @@ extract = "norm" label = "||gal match cout||" noise_threshold = 1e-6 order = 42 +class = "physics_converged" # Matching linear-solve residual ‖mat·cof − rmat‖/‖rmat‖ — health check (should stay ~machine eps). [quantities.gal_match_residual] @@ -115,6 +125,7 @@ extract = "value" label = "gal match residual" noise_threshold = 1e-10 order = 44 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -123,3 +134,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/gal_resistive_pe.toml b/regression-harness/cases/gal_resistive_pe.toml index f943e8d49..27d4b24d4 100644 --- a/regression-harness/cases/gal_resistive_pe.toml +++ b/regression-harness/cases/gal_resistive_pe.toml @@ -24,6 +24,7 @@ extract = "toml_key:ForceFreeStates.integrator" label = "integrator (Δ′ provenance)" noise_threshold = 0 order = 1 +class = "topological" # Surface locations / count (sanity) [quantities.pe_rational_q] @@ -33,6 +34,7 @@ extract = "all_real" label = "rational q values" noise_threshold = 1e-8 order = 10 +class = "physics_converged" # Island half-widths per rational surface — the headline driven quantity [quantities.pe_island_half_width] @@ -42,6 +44,7 @@ extract = "all_real" label = "island half-widths" noise_threshold = 1e-8 order = 20 +class = "physics_converged" # Chirikov overlap parameter per surface [quantities.pe_chirikov] @@ -51,6 +54,7 @@ extract = "all_real" label = "Chirikov parameter" noise_threshold = 1e-8 order = 21 +class = "physics_converged" # Resonant area-weighted field b̄^r = Φ^r/A^r per surface (tesla) — norm + per-surface complex values [quantities.pe_resonant_flux_norm] @@ -60,6 +64,7 @@ extract = "norm" label = "||resonant b field||" noise_threshold = 1e-6 order = 30 +class = "physics_converged" [quantities.pe_resonant_flux_all] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -68,6 +73,7 @@ extract = "all_complex" label = "resonant b field (all)" noise_threshold = 1e-8 order = 31 +class = "physics_converged" # Penetrated (reconnected) resonant field per surface — the resistive shielding signature [quantities.pe_penetrated_field] @@ -77,6 +83,7 @@ extract = "all_complex" label = "penetrated b field" noise_threshold = 1e-8 order = 32 +class = "physics_converged" # Per-surface driven-response Δ' from the PE singular-coupling jump (∂b^ψ/∂ψ across the # rational surface over 2π·χ₁, forced by the applied flux). This is a driven-response @@ -88,6 +95,7 @@ extract = "all_complex" label = "PE driven-response Δ′ (SingularCoupling jump)" noise_threshold = 1e-8 order = 33 +class = "physics_converged" # Resonant-field coupling matrix (n_rational × numpert_total) — Frobenius norm, catches broader drift [quantities.pe_C_resonant_flux_norm] @@ -97,6 +105,7 @@ extract = "norm" label = "||C resonant b field||" noise_threshold = 1e-6 order = 40 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -105,3 +114,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/ggj_ray_q500i.toml b/regression-harness/cases/ggj_ray_q500i.toml index f5722ee4d..3b33b87f4 100644 --- a/regression-harness/cases/ggj_ray_q500i.toml +++ b/regression-harness/cases/ggj_ray_q500i.toml @@ -22,6 +22,7 @@ extract = "value" label = "Re(Δ_odd)" noise_threshold = 1e-4 order = 10 +class = "physics_converged" [quantities.delta_odd_imag] h5path = "ggj/delta_odd_imag" @@ -30,6 +31,7 @@ extract = "value" label = "Im(Δ_odd)" noise_threshold = 1e-4 order = 11 +class = "physics_converged" [quantities.delta_even_real] h5path = "ggj/delta_even_real" @@ -38,6 +40,7 @@ extract = "value" label = "Re(Δ_even)" noise_threshold = 1e-4 order = 12 +class = "physics_converged" [quantities.delta_even_imag] h5path = "ggj/delta_even_imag" @@ -46,6 +49,7 @@ extract = "value" label = "Im(Δ_even)" noise_threshold = 1e-4 order = 13 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -54,3 +58,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/ggj_reference.toml b/regression-harness/cases/ggj_reference.toml index b35ca20a7..a8df484d9 100644 --- a/regression-harness/cases/ggj_reference.toml +++ b/regression-harness/cases/ggj_reference.toml @@ -15,6 +15,7 @@ extract = "value" label = "Re(Δ_odd)" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.delta_odd_imag] h5path = "ggj/delta_odd_imag" @@ -23,6 +24,7 @@ extract = "value" label = "Im(Δ_odd)" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.delta_even_real] h5path = "ggj/delta_even_real" @@ -31,6 +33,7 @@ extract = "value" label = "Re(Δ_even)" noise_threshold = 1e-10 order = 12 +class = "physics_converged" [quantities.delta_even_imag] h5path = "ggj/delta_even_imag" @@ -39,6 +42,7 @@ extract = "value" label = "Im(Δ_even)" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -47,3 +51,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/slayer_dels_fitzpatrick.toml b/regression-harness/cases/slayer_dels_fitzpatrick.toml index 0cede1614..c44dcb135 100644 --- a/regression-harness/cases/slayer_dels_fitzpatrick.toml +++ b/regression-harness/cases/slayer_dels_fitzpatrick.toml @@ -47,6 +47,7 @@ extract = "all_real" label = "Re(delta_s/d_beta) — Fitzpatrick fig 6.2" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.dels_db_im] h5path = "fitzpatrick/dels_db_im" @@ -55,6 +56,7 @@ extract = "all_real" label = "Im(delta_s/d_beta) — Fitzpatrick fig 6.3" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -63,3 +65,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/slayer_delta_probe.toml b/regression-harness/cases/slayer_delta_probe.toml index 41645a262..a42bb4c76 100644 --- a/regression-harness/cases/slayer_delta_probe.toml +++ b/regression-harness/cases/slayer_delta_probe.toml @@ -34,6 +34,7 @@ extract = "all_real" label = "SLAYER inner-layer Re Δ(Q) on the fixed grid" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.delta_im] h5path = "slayer_probe/Delta_im" @@ -42,6 +43,7 @@ extract = "all_real" label = "SLAYER inner-layer Im Δ(Q) on the fixed grid" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -50,3 +52,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_calculated.toml b/regression-harness/cases/solovev_kinetic_calculated.toml index d793afe9f..80d5b322e 100644 --- a/regression-harness/cases/solovev_kinetic_calculated.toml +++ b/regression-harness/cases/solovev_kinetic_calculated.toml @@ -16,6 +16,7 @@ type = "complex_vector" extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -23,6 +24,7 @@ type = "complex_vector" extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -30,6 +32,7 @@ type = "complex_vector" extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -37,6 +40,7 @@ type = "complex_vector" extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.et_all] @@ -45,6 +49,7 @@ type = "complex_vector" extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 +class = "physics_converged" # Integration [quantities.nstep] @@ -53,6 +58,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (saved)" noise_threshold = 0 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -60,6 +66,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (total)" noise_threshold = 0 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -68,6 +75,7 @@ type = "real_scalar" extract = "value" label = "q0" noise_threshold = 1e-12 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -75,6 +83,7 @@ type = "real_scalar" extract = "value" label = "q95" noise_threshold = 1e-12 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -83,6 +92,7 @@ type = "int_scalar" extract = "value" label = "# singular surfaces" noise_threshold = 0 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -90,6 +100,7 @@ type = "real_vector" extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -97,6 +108,7 @@ type = "real_vector" extract = "all_real" label = "singular q values" noise_threshold = 1e-8 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -105,6 +117,7 @@ type = "int_scalar" extract = "value" label = "mpert" noise_threshold = 0 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -112,6 +125,7 @@ type = "int_scalar" extract = "value" label = "npert" noise_threshold = 0 +class = "topological" # Runtime (special: not from H5) [quantities.runtime] @@ -120,3 +134,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" noise_threshold = 0.0 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_multiion.toml b/regression-harness/cases/solovev_kinetic_multiion.toml index faf23acfc..fd3285bf5 100644 --- a/regression-harness/cases/solovev_kinetic_multiion.toml +++ b/regression-harness/cases/solovev_kinetic_multiion.toml @@ -16,6 +16,7 @@ extract = "value" label = "NTV total (D+T+imp+e) [N·m]" noise_threshold = 1e-8 order = 10 +class = "physics_converged" [quantities.ntv_D] h5path = "KineticForces/PerSpecies/ion_z1_m2/fgar/total_torque" @@ -24,6 +25,7 @@ extract = "value" label = "NTV Deuterium [N·m]" noise_threshold = 1e-8 order = 11 +class = "physics_converged" [quantities.ntv_T] h5path = "KineticForces/PerSpecies/ion_z1_m3/fgar/total_torque" @@ -32,6 +34,7 @@ extract = "value" label = "NTV Tritium [N·m]" noise_threshold = 1e-8 order = 12 +class = "physics_converged" [quantities.ntv_electron] h5path = "KineticForces/PerSpecies/electron/fgar/total_torque" @@ -40,6 +43,7 @@ extract = "value" label = "NTV electron [N·m]" noise_threshold = 1e-8 order = 14 +class = "physics_converged" # Stability anchors — confirm the upstream FFS/PE stages feeding the NTV diagnostic. [quantities.et_real] @@ -49,6 +53,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.msing] h5path = "SingularSurfaces/rational_count" @@ -57,6 +62,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 21 +class = "topological" [quantities.runtime] h5path = "" @@ -65,3 +71,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 90 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_ntv.toml b/regression-harness/cases/solovev_kinetic_ntv.toml index 87f1d9667..6d59556a9 100644 --- a/regression-harness/cases/solovev_kinetic_ntv.toml +++ b/regression-harness/cases/solovev_kinetic_ntv.toml @@ -18,6 +18,7 @@ extract = "all_real" label = "NTV torque fgar [Re, Im]" noise_threshold = 1e-8 order = 10 +class = "physics_converged" [quantities.ntv_psi_nsteps] h5path = "KineticForces/fgar/psi_nsteps" @@ -36,6 +37,7 @@ extract = "real_first" label = "root-area-weighted total energy Re(et[1])" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.msing] h5path = "SingularSurfaces/rational_count" @@ -44,6 +46,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 21 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -52,6 +55,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 22 +class = "physics_converged" [quantities.q0] h5path = "Equilibrium/q_axis" @@ -60,6 +64,7 @@ extract = "value" label = "q0" noise_threshold = 0 order = 23 +class = "equilibrium_scalar" # Runtime (special: not from H5) [quantities.runtime] @@ -68,3 +73,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" noise_threshold = 0.0 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_kinetic_nuzero.toml b/regression-harness/cases/solovev_kinetic_nuzero.toml index deb449473..75958f31d 100644 --- a/regression-harness/cases/solovev_kinetic_nuzero.toml +++ b/regression-harness/cases/solovev_kinetic_nuzero.toml @@ -20,6 +20,7 @@ type = "complex_vector" extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -27,6 +28,7 @@ type = "complex_vector" extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -34,6 +36,7 @@ type = "complex_vector" extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -41,6 +44,7 @@ type = "complex_vector" extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.et_all] @@ -49,6 +53,7 @@ type = "complex_vector" extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 +class = "physics_converged" # Integration [quantities.nstep] @@ -57,6 +62,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (saved)" noise_threshold = 0 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -64,6 +70,7 @@ type = "int_scalar" extract = "value" label = "ODE steps (total)" noise_threshold = 0 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -72,6 +79,7 @@ type = "real_scalar" extract = "value" label = "q0" noise_threshold = 1e-12 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -79,6 +87,7 @@ type = "real_scalar" extract = "value" label = "q95" noise_threshold = 1e-12 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -87,6 +96,7 @@ type = "int_scalar" extract = "value" label = "# singular surfaces" noise_threshold = 0 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -94,6 +104,7 @@ type = "real_vector" extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -101,6 +112,7 @@ type = "real_vector" extract = "all_real" label = "singular q values" noise_threshold = 1e-8 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -109,6 +121,7 @@ type = "int_scalar" extract = "value" label = "mpert" noise_threshold = 0 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -116,6 +129,7 @@ type = "int_scalar" extract = "value" label = "npert" noise_threshold = 0 +class = "topological" # Runtime (special: not from H5) [quantities.runtime] @@ -124,3 +138,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" noise_threshold = 0.0 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_multi_n.toml b/regression-harness/cases/solovev_multi_n.toml index 2a6692540..e54bd353a 100644 --- a/regression-harness/cases/solovev_multi_n.toml +++ b/regression-harness/cases/solovev_multi_n.toml @@ -15,6 +15,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -23,6 +24,7 @@ extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.vacuum_eigenvalue] h5path = "ForceFreeStates/FreeBoundaryStability/vacuum_eigenvalue" @@ -31,6 +33,7 @@ extract = "value" label = "vacuum matrix min eigenvalue" noise_threshold = 1e-10 order = 14 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.ep_all] @@ -40,6 +43,7 @@ extract = "all_complex" label = "plasma energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ev_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -48,6 +52,7 @@ extract = "all_complex" label = "vacuum energy (all)" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -56,6 +61,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 22 +class = "physics_converged" # Integration [quantities.nstep] @@ -65,6 +71,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -73,6 +80,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 31 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -82,6 +90,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 40 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -90,6 +99,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 41 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -99,6 +109,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 50 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -107,6 +118,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 51 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -116,6 +128,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 60 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -124,6 +137,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 61 +class = "topological" # Profiles (checksums) [quantities.q_profile] @@ -133,6 +147,7 @@ extract = "checksum" label = "q profile (checksum)" noise_threshold = 0 order = 70 +class = "physics_converged" # Runtime [quantities.runtime] @@ -142,3 +157,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/solovev_n1.toml b/regression-harness/cases/solovev_n1.toml index d7f4224e5..758092343 100644 --- a/regression-harness/cases/solovev_n1.toml +++ b/regression-harness/cases/solovev_n1.toml @@ -15,6 +15,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_imag] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -23,6 +24,7 @@ extract = "imag_first" label = "total energy Im(et[1])" noise_threshold = 1e-10 order = 11 +class = "physics_converged" [quantities.ep_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_plasma_energies" @@ -31,6 +33,7 @@ extract = "real_first" label = "plasma energy Re(ep[1])" noise_threshold = 1e-10 order = 12 +class = "physics_converged" [quantities.ev_real] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -39,6 +42,7 @@ extract = "real_first" label = "vacuum energy Re(ev[1])" noise_threshold = 1e-10 order = 13 +class = "physics_converged" [quantities.vacuum_eigenvalue] h5path = "ForceFreeStates/FreeBoundaryStability/vacuum_eigenvalue" @@ -47,6 +51,7 @@ extract = "value" label = "vacuum matrix min eigenvalue" noise_threshold = 1e-10 order = 14 +class = "physics_converged" # Energies — full eigenvalue arrays [quantities.ep_all] @@ -56,6 +61,7 @@ extract = "all_complex" label = "plasma energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" [quantities.ev_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_vacuum_energies" @@ -64,6 +70,7 @@ extract = "all_complex" label = "vacuum energy (all)" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -72,6 +79,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 22 +class = "physics_converged" # Integration [quantities.nstep] @@ -81,6 +89,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" [quantities.nstep_total] h5path = "ForceFreeStates/Solutions/ForwardIntegration/nstep_total" @@ -89,6 +98,7 @@ extract = "value" label = "ODE steps (total)" noise_threshold = 0 order = 31 +class = "diagnostic" # Equilibrium [quantities.q0] @@ -98,6 +108,7 @@ extract = "value" label = "q0" noise_threshold = 1e-12 order = 40 +class = "equilibrium_scalar" [quantities.q95] h5path = "Equilibrium/q_95" @@ -106,6 +117,7 @@ extract = "value" label = "q95" noise_threshold = 1e-12 order = 41 +class = "equilibrium_scalar" [quantities.betat] h5path = "Equilibrium/beta_t" @@ -114,6 +126,7 @@ extract = "value" label = "beta_t" noise_threshold = 1e-12 order = 42 +class = "equilibrium_scalar" [quantities.betan] h5path = "Equilibrium/beta_N" @@ -122,6 +135,7 @@ extract = "value" label = "beta_n" noise_threshold = 1e-12 order = 43 +class = "equilibrium_scalar" # Singular surfaces [quantities.msing] @@ -131,6 +145,7 @@ extract = "value" label = "# singular surfaces" noise_threshold = 0 order = 50 +class = "topological" [quantities.sing_psi] h5path = "SingularSurfaces/rational_psi" @@ -139,6 +154,7 @@ extract = "all_real" label = "singular psi locations" noise_threshold = 1e-8 order = 51 +class = "physics_converged" [quantities.sing_q] h5path = "SingularSurfaces/rational_q" @@ -147,6 +163,7 @@ extract = "all_real" label = "singular q values" noise_threshold = 1e-8 order = 52 +class = "physics_converged" # Mode numbers [quantities.mpert] @@ -156,6 +173,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 60 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -164,6 +182,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 61 +class = "topological" # Profiles (checksums for fast comparison of large arrays) [quantities.q_profile] @@ -173,6 +192,7 @@ extract = "checksum" label = "q profile (checksum)" noise_threshold = 0 order = 70 +class = "physics_converged" [quantities.pressure_profile] h5path = "Equilibrium/Profiles/mu0p" @@ -181,6 +201,7 @@ extract = "checksum" label = "pressure profile (checksum)" noise_threshold = 0 order = 71 +class = "physics_converged" # Bitwise reproducibility guard for the ideal asymptotic coefficients (issue: kinetic # runs used to dump uninitialized memory here; ideal values must stay byte-stable). @@ -191,6 +212,7 @@ extract = "checksum" label = "ca_left (checksum)" noise_threshold = 0 order = 72 +class = "physics_converged" # Runtime (special: not from H5) [quantities.runtime] @@ -200,3 +222,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/src/config.jl b/regression-harness/src/config.jl index c1a50a4a2..a0c9b660f 100644 --- a/regression-harness/src/config.jl +++ b/regression-harness/src/config.jl @@ -14,6 +14,9 @@ function load_case(filepath::String)::CaseSpec quantities = QuantitySpec[] if haskey(data, "quantities") for (qty_name, qty_data) in data["quantities"] + # Every quantity declares its golden tolerance class; there is no inference from its name. + haskey(qty_data, "class") || error("$filepath: quantity '$qty_name' has no `class`; declare one of $(join(TOLERANCE_CLASSES, ", "))") + qty_data["class"] in TOLERANCE_CLASSES || error("$filepath: quantity '$qty_name' has unknown class '$(qty_data["class"])'") push!( quantities, QuantitySpec( @@ -24,7 +27,7 @@ function load_case(filepath::String)::CaseSpec get(qty_data, "label", qty_name), get(qty_data, "noise_threshold", 1e-10), get(qty_data, "order", 1000), - get(qty_data, "class", "") + qty_data["class"] ) ) end diff --git a/regression-harness/src/golden.jl b/regression-harness/src/golden.jl index ec7b0c037..d43988f7d 100644 --- a/regression-harness/src/golden.jl +++ b/regression-harness/src/golden.jl @@ -28,8 +28,9 @@ How strictly a quantity must reproduce, and why. - `unconverged` — measured and found to have no plateau; tracked differentially, never pinned, so that a known-unconverged quantity is visibly excluded instead of quietly given a wide bound -A quantity's class comes from its case file (an explicit `class` key, else `infer_class`), never -from the golden file, so demoting a gate to a non-gating class is a reviewed case-file change. +A quantity's class comes only from the `class` key its case file must declare, never from the +golden file or the quantity's name, so demoting a gate to a non-gating class is a reviewed +case-file change and renaming a quantity cannot silently change its class. A gating entry's `atol` must be 0 unless its golden value contains an exact zero (a scalar equal to 0, or an array element whose every component is 0). Everywhere else the relative test is the whole @@ -274,30 +275,15 @@ function save_golden(meta::GoldenMeta, values::Dict{String,GoldenValue}) end """ -Classify a quantity from its case spec, so a new case gets sensible defaults without every -tolerance having to be written by hand. +Return a quantity's declared tolerance class, validated against `TOLERANCE_CLASSES`. -An explicit `class` in the case file wins. Otherwise integer counts are topological; runtimes and -step counts describe the numerics rather than the physics and so are diagnostic; everything else -is assumed to be a physics quantity that must be converged, which is the conservative assumption — -it gates. +There is no inference from the quantity's name or type: a missing class is an error, so that +every gate is an explicit, reviewed choice in the case file. """ function infer_class(spec::QuantitySpec)::String - if !isempty(spec.class) - spec.class in TOLERANCE_CLASSES || error("Quantity '$(spec.name)': unknown class '$(spec.class)' in its case file") - return spec.class - end - spec.type == "runtime" && return "diagnostic" - spec.name in ("nstep", "nstep_total") && return "diagnostic" - spec.type == "int_scalar" && return "topological" - # A control token has no meaningful tolerance between "riccati" and "galerkin". - startswith(spec.extract, "toml_key:") && return "topological" - name = spec.name - # sing_psi / sing_q are absent on purpose: they come from a root search, not pure quadrature. - equilibrium_names = ("q0", "q95", "betat", "betan", "betap1", "betap2", "betap3", "betaj", - "li1", "li2", "li3", "volume", "crnt", "bt0", "bwall", "aratio", "kappa", "psio") - name in equilibrium_names && return "equilibrium_scalar" - return "physics_converged" + isempty(spec.class) && error("Quantity '$(spec.name)': no `class` declared in its case file") + spec.class in TOLERANCE_CLASSES || error("Quantity '$(spec.name)': unknown class '$(spec.class)' in its case file") + return spec.class end """ diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl index 08bc9db99..231695abc 100644 --- a/regression-harness/src/types.jl +++ b/regression-harness/src/types.jl @@ -5,8 +5,7 @@ Shared data structures for the regression harness. """ Specification for a single quantity to extract from gpec.h5. -`class` is the golden tolerance class declared in the case file, or empty to let `infer_class` -choose; declaring it is the only way to make a quantity non-gating that inference would gate. +`class` is the golden tolerance class declared in the case file; every quantity must declare one. """ struct QuantitySpec name::String @@ -17,7 +16,7 @@ struct QuantitySpec label::String # Human-readable label for reports noise_threshold::Float64 # Absolute changes below this are noise order::Int # Display order in reports (lower = earlier) - class::String # Declared golden tolerance class; empty = inferred + class::String # Declared golden tolerance class (one of TOLERANCE_CLASSES) end """ diff --git a/regression-harness/test/runtests.jl b/regression-harness/test/runtests.jl index e28f8bd53..7a658bf2b 100644 --- a/regression-harness/test/runtests.jl +++ b/regression-harness/test/runtests.jl @@ -18,7 +18,7 @@ for f in ("types.jl", "env.jl", "config.jl", "database.jl", "utils.jl", "extract include(joinpath(HARNESS_DIR, "src", f)) end -qspec(name; type="real_scalar", extract="value", label=name, noise=0.0, order=1, class="") = +qspec(name; type="real_scalar", extract="value", label=name, noise=0.0, order=1, class="physics_converged") = QuantitySpec(name, "", type, extract, label, noise, order, class) golden_val(name; value_type="real", value_real=nothing, value_int=nothing, value_text=nothing, @@ -200,7 +200,7 @@ extracted_q(name; value_real=nothing, value_int=nothing, value_text=nothing, val extracted_q("x"; value_text="abc123", value_type="checksum"), extracted_q("rt"; value_real=12.5), extracted_q("unspecced"; value_real=1.0)], - [spec, qspec("rt"; type="runtime")], nothing) + [spec, qspec("rt"; type="runtime", class="diagnostic")], nothing) @test isempty(vals3) end @@ -424,21 +424,37 @@ value = 12.5 end @testset "infer_class" begin - @test infer_class(qspec("runtime"; type="runtime")) == "diagnostic" - @test infer_class(qspec("nstep"; type="int_scalar")) == "diagnostic" - @test infer_class(qspec("msing"; type="int_scalar")) == "topological" - @test infer_class(qspec("integrator"; type="token", extract="toml_key:ForceFreeStates.integrator")) == "topological" - @test infer_class(qspec("q0")) == "equilibrium_scalar" - @test infer_class(qspec("psio")) == "equilibrium_scalar" - @test infer_class(qspec("et1_re")) == "physics_converged" - # A class declared in the case file wins over inference, and must be a real class. - @test infer_class(qspec("ntv_psi_nsteps"; type="int_scalar", class="diagnostic")) == "diagnostic" + # The declared class is returned as-is, whatever the quantity's name or type suggests. + for class in TOLERANCE_CLASSES + @test infer_class(qspec("x"; class=class)) == class + end + @test infer_class(qspec("q0"; class="physics_converged")) == "physics_converged" + @test infer_class(qspec("nstep"; type="int_scalar", class="topological")) == "topological" + # A missing or unknown class is an error, never a default. + @test_throws ErrorException infer_class(qspec("x"; class="")) @test_throws ErrorException infer_class(qspec("x"; class="gating_optional")) end + @testset "load_case requires a declared class" begin + dir = mktempdir() + write_case(body) = (path = joinpath(dir, "c.toml"); write(path, "[case]\nname = \"c\"\n\n" * body); path) + block = "[quantities.a]\nh5path = \"x\"\ntype = \"real_scalar\"\nextract = \"value\"\n" + @test load_case(write_case(block * "class = \"diagnostic\"\n")).quantities[1].class == "diagnostic" + err = try + load_case(write_case(block)) + "" + catch e + sprint(showerror, e) + end + @test occursin("c.toml", err) && occursin("'a'", err) && occursin("no `class`", err) + @test_throws ErrorException load_case(write_case(block * "class = \"gating_optional\"\n")) + # Every committed case declares a valid class on every quantity. + @test all(q -> q.class in TOLERANCE_CLASSES, Iterators.flatten(c.quantities for c in values(load_all_cases(CASES_DIR)))) + end + @testset "is_pinnable" begin @test is_pinnable(qspec("a")) - @test !is_pinnable(qspec("rt"; type="runtime")) + @test !is_pinnable(qspec("rt"; type="runtime", class="diagnostic")) @test !is_pinnable(qspec("h"; extract="checksum")) end @@ -447,8 +463,8 @@ value = 12.5 case = CaseSpec("count_case", "synthetic", "", [ qspec("a"), - qspec("rt"; type="runtime", extract=""), - qspec("c"; type="int_scalar"), + qspec("rt"; type="runtime", extract="", class="diagnostic"), + qspec("c"; type="int_scalar", class="topological"), qspec("d"; class="diagnostic"), qspec("e"), qspec("f"), From 2d8099e95f928ff37d677ae2f78a4562374c9c1a Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 15:38:32 -0400 Subject: [PATCH 27/30] CI - FEATURE - Run the regression harness unit tests on harness changes The harness unit tests load only the harness project, not GPEC, so they run in minutes on any pull request or push touching regression-harness/. Co-Authored-By: Claude Opus 5.5 --- .../workflows/regression-harness-unit.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/regression-harness-unit.yaml diff --git a/.github/workflows/regression-harness-unit.yaml b/.github/workflows/regression-harness-unit.yaml new file mode 100644 index 000000000..950ea51d5 --- /dev/null +++ b/.github/workflows/regression-harness-unit.yaml @@ -0,0 +1,46 @@ +name: Regression Harness Unit Tests + +# Unit tests for the regression harness's own logic (golden values, case loading), on +# synthetic data only. They load the harness project, not GPEC, so they run in minutes. +on: + pull_request: + branches: + - main + - develop + paths: + - 'regression-harness/**' + - '.github/workflows/regression-harness-unit.yaml' + push: + branches: + - main + - develop + paths: + - 'regression-harness/**' + - '.github/workflows/regression-harness-unit.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + unit: + name: regression-harness unit tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Julia + uses: julia-actions/setup-julia@v2 + with: + version: '1.11' + + - name: Instantiate the harness environment + run: julia --project=regression-harness -e 'using Pkg; Pkg.instantiate()' + + - name: Run the harness unit tests + run: julia --project=regression-harness regression-harness/test/runtests.jl From 99170b75b7070659308c554ad6330956683a2b0e Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 15:42:40 -0400 Subject: [PATCH 28/30] Regression - MINOR - Parse --accept-exceeding with the other CLI options Co-Authored-By: Claude Opus 5.5 --- regression-harness/regress.jl | 13 +++--- regression-harness/src/database.jl | 70 +++++++++++++++-------------- regression-harness/src/types.jl | 10 +++-- regression-harness/test/runtests.jl | 2 +- 4 files changed, 52 insertions(+), 43 deletions(-) diff --git a/regression-harness/regress.jl b/regression-harness/regress.jl index 7d9988684..f6f890e40 100644 --- a/regression-harness/regress.jl +++ b/regression-harness/regress.jl @@ -34,6 +34,7 @@ function parse_args(args) fail_on_change = false check = false update_golden = false + accept_exceeding = false reason = nothing help = false @@ -70,6 +71,9 @@ function parse_args(args) elseif arg == "--update-golden" update_golden = true i += 1 + elseif arg == "--accept-exceeding" + accept_exceeding = true + i += 1 elseif arg == "--reason" && i < length(args) reason = args[i+1] i += 2 @@ -97,7 +101,7 @@ function parse_args(args) end return CLIOptions(cases, refs, ref_range, force, list_cases, show_qty, show_case, db_path, verbose, - no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, check, update_golden, reason, help) + no_instantiate, no_pin_manifest, allow_env_mismatch, fail_on_change, check, update_golden, accept_exceeding, reason, help) end const HELP_TEXT = """ @@ -194,8 +198,7 @@ Examples: function main(args=ARGS) # Kept out of CLIOptions so this flag needs no change to the shared option struct. - accept_exceeding = "--accept-exceeding" in args - opts = parse_args(filter(!=("--accept-exceeding"), args)) + opts = parse_args(args) if opts.help print(HELP_TEXT) @@ -239,7 +242,7 @@ function main(args=ARGS) error("--update-golden requires --reason \"...\": a golden change is a claim about physics, " * "and the reviewer needs to know what changed and why") end - if accept_exceeding && !opts.update_golden + if opts.accept_exceeding && !opts.update_golden error("--accept-exceeding only applies to --update-golden") end if opts.update_golden && isempty(opts.cases) @@ -306,7 +309,7 @@ function main(args=ARGS) if opts.update_golden update_golden_from_run(db, case_spec, resolved_refs[1].commit_hash, - String(opts.reason), REPO_ROOT; accept_exceeding=accept_exceeding) + String(opts.reason), REPO_ROOT; accept_exceeding=opts.accept_exceeding) elseif opts.check summary = report_golden_check(db, case_spec, resolved_refs[1].commit_hash) n_golden_fail += summary.n_fail diff --git a/regression-harness/src/database.jl b/regression-harness/src/database.jl index b174e3074..3a5cecb77 100644 --- a/regression-harness/src/database.jl +++ b/regression-harness/src/database.jl @@ -50,7 +50,9 @@ added by a later schema migration (NULL on every pre-existing row) need both cas """ _column(x, default) = (x === nothing || x === missing) ? default : x -"""Materialize SQLite query results as a Vector of NamedTuples.""" +""" +Materialize SQLite query results as a Vector of NamedTuples. +""" function query_rows(db::SQLite.DB, sql::String, params=()) result = DBInterface.execute(db, sql, params) ct = Tables.columntable(result) @@ -70,7 +72,9 @@ const ENV_COLUMNS = [ ("pinned", "INTEGER") ] -"""Add any `runs` columns missing from a database created by an earlier harness version.""" +""" +Add any `runs` columns missing from a database created by an earlier harness version. +""" function migrate_schema!(db::SQLite.DB) existing = Set(String[]) for row in query_rows(db, "PRAGMA table_info(runs)") @@ -108,10 +112,10 @@ environment. Rows predating fingerprinting hold NULL and therefore never match entries most likely to be misleading are exactly the ones that get re-run. """ function is_cached(db::SQLite.DB, commit_hash::String, case_name::String; - expected_key::Union{String,Nothing}=nothing)::Bool + expected_key::Union{String,Nothing}=nothing)::Bool if expected_key === nothing rows = query_rows(db, "SELECT id FROM runs WHERE commit_hash = ? AND case_name = ? AND success = 1", - (commit_hash, case_name)) + (commit_hash, case_name)) return !isempty(rows) end rows = query_rows(db, @@ -126,7 +130,7 @@ fingerprinting. Used to explain *why* a cached result was rejected. """ function cached_env_key(db::SQLite.DB, commit_hash::String, case_name::String)::Union{String,Nothing} rows = query_rows(db, "SELECT env_key FROM runs WHERE commit_hash = ? AND case_name = ?", - (commit_hash, case_name)) + (commit_hash, case_name)) isempty(rows) && return nothing key = _column(first(rows).env_key, nothing) key === nothing && return nothing @@ -136,15 +140,15 @@ end function delete_cached(db::SQLite.DB, commit_hash::String, case_name::String) # ON DELETE CASCADE handles quantities cleanup automatically DBInterface.execute(db, "DELETE FROM runs WHERE commit_hash = ? AND case_name = ?", - (commit_hash, case_name)) + (commit_hash, case_name)) end function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::AbstractString, - commit_date::AbstractString, commit_msg::AbstractString, - case_name::AbstractString, runtime_s::Float64, - extracted::Vector{ExtractedQuantity}; - success::Bool=true, error_msg::AbstractString="", - fingerprint::EnvFingerprint=UNKNOWN_ENV) + commit_date::AbstractString, commit_msg::AbstractString, + case_name::AbstractString, runtime_s::Float64, + extracted::Vector{ExtractedQuantity}; + success::Bool=true, error_msg::AbstractString="", + fingerprint::EnvFingerprint=UNKNOWN_ENV) ran_at = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS") SQLite.transaction(db) do @@ -156,10 +160,10 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs env_key, julia_version, os_arch, manifest_sha, nthreads, blas_threads, pinned) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (String(commit_hash), String(commit_short), String(commit_date), String(commit_msg), - String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg), - env_key(fingerprint), fingerprint.julia_version, fingerprint.os_arch, - fingerprint.manifest_sha, fingerprint.nthreads, fingerprint.blas_threads, - fingerprint.pinned ? 1 : 0)) + String(case_name), ran_at, runtime_s, success ? 1 : 0, String(error_msg), + env_key(fingerprint), fingerprint.julia_version, fingerprint.os_arch, + fingerprint.manifest_sha, fingerprint.nthreads, fingerprint.blas_threads, + fingerprint.pinned ? 1 : 0)) run_id = SQLite.last_insert_rowid(db) @@ -169,14 +173,14 @@ function store_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::Abs (run_id, qty_name, qty_label, value_real, value_int, value_text, value_type, noise_threshold) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", (run_id, eq.name, eq.label, eq.value_real, eq.value_int, eq.value_text, - eq.value_type, eq.noise_threshold)) + eq.value_type, eq.noise_threshold)) end end end function store_failed_run(db::SQLite.DB, commit_hash::AbstractString, commit_short::AbstractString, - commit_date::AbstractString, commit_msg::AbstractString, - case_name::AbstractString, error_msg::AbstractString) + commit_date::AbstractString, commit_msg::AbstractString, + case_name::AbstractString, error_msg::AbstractString) ran_at = Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS") delete_cached(db, String(commit_hash), String(case_name)) DBInterface.execute(db, @@ -184,14 +188,14 @@ function store_failed_run(db::SQLite.DB, commit_hash::AbstractString, commit_sho (commit_hash, commit_short, commit_date, commit_msg, case_name, ran_at, runtime_s, success, error_msg) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", (String(commit_hash), String(commit_short), String(commit_date), String(commit_msg), - String(case_name), ran_at, 0.0, 0, String(error_msg))) + String(case_name), ran_at, 0.0, 0, String(error_msg))) end """ Get all quantities for a (commit, case) pair. Returns Dict{qty_name => NamedTuple}. """ function get_quantities(db::SQLite.DB, commit_hash::String, case_name::String) - results = Dict{String, NamedTuple}() + results = Dict{String,NamedTuple}() rows = query_rows(db, """SELECT q.qty_name, q.qty_label, q.value_real, q.value_int, q.value_text, q.value_type, q.noise_threshold @@ -203,12 +207,12 @@ function get_quantities(db::SQLite.DB, commit_hash::String, case_name::String) for row in rows name = something(row.qty_name, "") results[name] = ( - label = something(row.qty_label, name), - value_real = row.value_real, - value_int = row.value_int, - value_text = row.value_text, - value_type = something(row.value_type, "missing"), - noise_threshold = something(row.noise_threshold, 0.0), + label=something(row.qty_label, name), + value_real=row.value_real, + value_int=row.value_int, + value_text=row.value_text, + value_type=something(row.value_type, "missing"), + noise_threshold=something(row.noise_threshold, 0.0) ) end return results @@ -234,13 +238,13 @@ function get_run_info(db::SQLite.DB, commit_hash::String, case_name::String) _column(row.pinned, 0) == 1 ) return ( - commit_short = something(row.commit_short, ""), - commit_date = something(row.commit_date, ""), - commit_msg = something(row.commit_msg, ""), - runtime_s = something(row.runtime_s, 0.0), - success = coalesce(row.success, 0) == 1, - error_msg = something(row.error_msg, ""), - fingerprint = fingerprint, + commit_short=something(row.commit_short, ""), + commit_date=something(row.commit_date, ""), + commit_msg=something(row.commit_msg, ""), + runtime_s=something(row.runtime_s, 0.0), + success=coalesce(row.success, 0) == 1, + error_msg=something(row.error_msg, ""), + fingerprint=fingerprint ) end diff --git a/regression-harness/src/types.jl b/regression-harness/src/types.jl index 231695abc..843d2308a 100644 --- a/regression-harness/src/types.jl +++ b/regression-harness/src/types.jl @@ -59,10 +59,11 @@ worktree (pinning is on by default, so that two refs differ only by source code) `allow_env_mismatch` lets a cached result from a different environment be reused instead of re-run. `fail_on_change` turns any changed quantity into a non-zero exit status, for CI use. -`check` runs the working tree and compares it against the committed golden values, which is the -mode CI gates on. `update_golden` regenerates those values from a fresh run; `reason` records -why, and is mandatory because a golden change is a claim about physics that a reviewer has to be -able to evaluate. +`check` runs the working tree and compares it against the committed golden values. +`update_golden` regenerates those values from a fresh run; `reason` records why, and is mandatory +because a golden change is a claim about physics that a reviewer has to be able to evaluate. +`accept_exceeding` lets such a re-pin move a gating quantity outside its old tolerance, recording +the previous value in the golden file. """ struct CLIOptions cases::Vector{String} @@ -80,6 +81,7 @@ struct CLIOptions fail_on_change::Bool check::Bool update_golden::Bool + accept_exceeding::Bool reason::Union{String,Nothing} help::Bool end diff --git a/regression-harness/test/runtests.jl b/regression-harness/test/runtests.jl index 7a658bf2b..6906a8705 100644 --- a/regression-harness/test/runtests.jl +++ b/regression-harness/test/runtests.jl @@ -413,7 +413,7 @@ value = 12.5 update_golden_from_run(db, case, sha, "moved", repo) end @test load_golden("exceed_case").meta.golden_version == 3 - @test_logs (:warn, r"beyond its old tolerance") match_mode=:any redirect_stdout(devnull) do + @test_logs (:warn, r"beyond its old tolerance") match_mode = :any redirect_stdout(devnull) do update_golden_from_run(db, case, sha, "moved", repo; accept_exceeding=true) end back = load_golden("exceed_case") From b5e92f4a88b129b205a900b16d2ea65e3f1c381a Mon Sep 17 00:00:00 2001 From: d-burg Date: Fri, 25 Sep 2026 15:43:00 -0400 Subject: [PATCH 29/30] InnerLayer.SLAYER - DOCS - Move the chi_tor docstring correction to its own pull request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The χ_φ description of chi_tor is a physics-docs change reviewed separately; this restores the three docstrings to develop's text so the golden-values PR touches only the harness. Co-Authored-By: Claude Opus 5.5 --- src/InnerLayer/SLAYER/LayerInputs.jl | 6 ++---- src/InnerLayer/SLAYER/LayerParameters.jl | 7 ++----- src/Tearing/Runner/Control.jl | 11 ++++------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/InnerLayer/SLAYER/LayerInputs.jl b/src/InnerLayer/SLAYER/LayerInputs.jl index d42e7b190..252e81ae9 100644 --- a/src/InnerLayer/SLAYER/LayerInputs.jl +++ b/src/InnerLayer/SLAYER/LayerInputs.jl @@ -183,10 +183,8 @@ profiles, without an intermediate file round-trip. - `zeff` -- effective charge (default `1.0`). - `chi_perp` -- perpendicular heat diffusivity [m²/s]. Scalar or a callable of `psi` (default `1.0`). - - `chi_tor` -- Fitzpatrick's χ_φ: anomalous perpendicular ion MOMENTUM - diffusivity `[m²/s]` (a viscosity entering the vorticity and parallel-flow - equations — not a heat diffusivity). Scalar or a callable of `psi` - (default `1.0`). + - `chi_tor` -- toroidal heat diffusivity [m²/s]. Scalar or a callable + of `psi` (default `1.0`). - `dr_val` -- resistive interchange index `D_R = E + F + H²` (Glasser-Greene-Johnson 1975) feeding the critical-Δ formulas (`:lar`, `:rfitzp`, `:toroidal`). When `nothing` (default), Julia diff --git a/src/InnerLayer/SLAYER/LayerParameters.jl b/src/InnerLayer/SLAYER/LayerParameters.jl index 064cb6312..ae875dea4 100644 --- a/src/InnerLayer/SLAYER/LayerParameters.jl +++ b/src/InnerLayer/SLAYER/LayerParameters.jl @@ -29,7 +29,7 @@ de-normalization. The parametrization uses `P_perp`, `P_tor`, and | `c_beta` | Compressibility √(β_local / (1 + β_local)) | | `D_norm` | (d_β/r_s) · S^(1/3) · √ι_e (Fitzpatrick normalized scale) | | `P_perp` | Perpendicular Prandtl number τ_R / τ_⊥ | -| `P_tor` | Momentum (magnetic) Prandtl number τ_R / τ_φ, τ_φ = r_s²/χ_φ | +| `P_tor` | Toroidal-direction Prandtl number τ_R / τ_‖tor | | `Q_e` | Normalized electron diamagnetic: −tauk · ω_*e | | `Q_i` | Normalized ion diamagnetic: −tauk · ω_*i | | `iota_e` | Q_e / (Q_e − Q_i) | @@ -210,10 +210,7 @@ parametrization (P_perp/P_tor/D_norm; the older magnetic/electron Prandtl - `R0` -- major radius [m] - `mu_i` -- ion mass in proton-mass units (e.g. 2.0 for D) - `zeff` -- effective charge - - `chi_perp` -- perpendicular energy diffusivity χ⊥ [m²/s] - - `chi_tor` -- Fitzpatrick's χ_φ, the anomalous perpendicular ion momentum - diffusivity `[m²/s]` (a viscosity, not heat; TJ Layer.tex eqs. — enters the - vorticity and parallel-flow equations, τ_φ = r_s²/χ_φ) + - `chi_perp`, `chi_tor` -- perpendicular / toroidal heat diffusivity [m²/s] - `m`, `n` -- poloidal / toroidal mode numbers at the surface - `dr_val`, `dgeo_val` -- inputs for the critical-Δ formula - `dc_type` -- one of `:none`, `:lar`, `:rfitzp`, `:toroidal` diff --git a/src/Tearing/Runner/Control.jl b/src/Tearing/Runner/Control.jl index f41777468..1a12ff7e6 100644 --- a/src/Tearing/Runner/Control.jl +++ b/src/Tearing/Runner/Control.jl @@ -34,13 +34,10 @@ constructor. callable of `psi` overrides it - `mu_i` -- ion mass in proton-mass units (default 2.0 for D) - `zeff` -- effective charge - - `chi_perp`, `chi_tor` -- fallback scalars [m²/s], used only when the - kinetic file carries no usable `chi_e`/`chi_phi` profile (dataset absent - or all-zero); otherwise the file's χ⊥(ψ)/χ_φ(ψ) take precedence. - `chi_perp` is the perpendicular ENERGY diffusivity χ⊥; `chi_tor` is - Fitzpatrick's χ_φ, the anomalous perpendicular ion MOMENTUM diffusivity - (a viscosity: it enters the vorticity and parallel-flow equations, TJ - Layer.tex — not a heat diffusivity) + - `chi_perp`, `chi_tor` -- fallback perpendicular / toroidal heat + diffusivity [m²/s], used only when the kinetic file carries no usable + `chi_e`/`chi_phi` profile (dataset absent or all-zero); otherwise the + file's χ⊥(ψ)/χ_φ(ψ) take precedence - `dr_val`, `dgeo_val` -- critical-Δ formula inputs. `nothing` (default) auto-derives them from the equilibrium: `dr_val` from the resistive interchange index `D_R = E + F + H²` at each surface, `dgeo_val` from the From 33478cdfe23a6ed6119ff7c79de3d336b2a489e4 Mon Sep 17 00:00:00 2001 From: d-burg Date: Sat, 3 Oct 2026 14:32:21 -0400 Subject: [PATCH 30/30] Regression - MINOR - Declare the tolerance class on the quantities develop added The merge brings in diiid_error_field, diiid_multi_n and five new diiid_n1 quantities, none of which declare the class this branch requires. Each gets the class the previous inference rule would have returned (runtime and step counts diagnostic, integer scalars topological, everything else physics_converged), as for the existing cases. Co-Authored-By: Claude Opus 5.5 --- .../cases/diiid_error_field.toml | 9 +++++++++ regression-harness/cases/diiid_multi_n.toml | 20 +++++++++++++++++++ regression-harness/cases/diiid_n1.toml | 5 +++++ 3 files changed, 34 insertions(+) diff --git a/regression-harness/cases/diiid_error_field.toml b/regression-harness/cases/diiid_error_field.toml index 2fd44b92a..4366f8483 100644 --- a/regression-harness/cases/diiid_error_field.toml +++ b/regression-harness/cases/diiid_error_field.toml @@ -16,6 +16,7 @@ extract = "all_real" label = "EFC overlap per kAt" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.efc_torque_full] h5path = "ErrorFields/NTV/torque_full_per_kat2" @@ -24,6 +25,7 @@ extract = "all_real" label = "EFC whole-field NTV torque per kAt² [N·m]" noise_threshold = 1e-8 order = 11 +class = "physics_converged" [quantities.efc_torque_residual] h5path = "ErrorFields/NTV/torque_residual_per_kat2" @@ -32,6 +34,7 @@ extract = "all_real" label = "EFC residual-field NTV torque per kAt² [N·m]" noise_threshold = 1e-8 order = 12 +class = "physics_converged" # The torque-versus-rotation table: reference rotation, the adaptive shift grid and the residual torque on it. [quantities.efc_omega_reference] @@ -41,6 +44,7 @@ extract = "all_real" label = "EFC reference rotation ω_ref [rad/s]" noise_threshold = 1e-6 order = 13 +class = "physics_converged" [quantities.efc_rotation_shift] h5path = "ErrorFields/NTV/RotationScan/rotation_shift" @@ -49,6 +53,7 @@ extract = "all_real" label = "EFC rotation-scan shifts [rad/s]" noise_threshold = 1e-6 order = 14 +class = "physics_converged" [quantities.efc_torque_residual_scan] h5path = "ErrorFields/NTV/RotationScan/torque_residual" @@ -57,6 +62,7 @@ extract = "all_real" label = "EFC residual torque against rotation [N·m/kAt²]" noise_threshold = 1e-8 order = 15 +class = "physics_converged" # Anchors of the stages feeding the couplings. [quantities.dominant_singular_values] @@ -66,6 +72,7 @@ extract = "all_real" label = "dominant-coupling singular values" noise_threshold = 1e-8 order = 20 +class = "physics_converged" [quantities.ef_delta_nominal] h5path = "ErrorFields/CoilSensitivities/DominantMode/delta_nominal" @@ -74,6 +81,7 @@ extract = "abs_first" label = "|delta_nominal| of coil set 1" noise_threshold = 1e-10 order = 21 +class = "physics_converged" [quantities.runtime] h5path = "" @@ -81,3 +89,4 @@ type = "runtime" extract = "value" label = "Runtime (s)" order = 99 +class = "diagnostic" diff --git a/regression-harness/cases/diiid_multi_n.toml b/regression-harness/cases/diiid_multi_n.toml index ca363248f..405234124 100644 --- a/regression-harness/cases/diiid_multi_n.toml +++ b/regression-harness/cases/diiid_multi_n.toml @@ -16,6 +16,7 @@ extract = "real_first" label = "total energy Re(et[1])" noise_threshold = 1e-10 order = 10 +class = "physics_converged" [quantities.et_all] h5path = "ForceFreeStates/FreeBoundaryStability/eigenmode_energies" @@ -24,6 +25,7 @@ extract = "all_complex" label = "total energy (all)" noise_threshold = 1e-10 order = 20 +class = "physics_converged" # Integration [quantities.nstep] @@ -33,6 +35,7 @@ extract = "value" label = "ODE steps (saved)" noise_threshold = 0 order = 30 +class = "diagnostic" # Mode space [quantities.mpert] @@ -42,6 +45,7 @@ extract = "value" label = "mpert" noise_threshold = 0 order = 40 +class = "topological" [quantities.npert] h5path = "Info/npert" @@ -50,6 +54,7 @@ extract = "value" label = "npert" noise_threshold = 0 order = 41 +class = "topological" [quantities.mode_n] h5path = "PerturbedEquilibrium/mode_n" @@ -58,6 +63,7 @@ extract = "checksum" label = "toroidal mode label of each mode index (checksum)" noise_threshold = 0 order = 42 +class = "physics_converged" # Control-surface response, one block per n (b~ space) [quantities.permeability_diag] @@ -67,6 +73,7 @@ extract = "diagonal_complex" label = "permeability diagonal P~_ii (all n blocks)" noise_threshold = 1e-8 order = 50 +class = "physics_converged" [quantities.forcing_b_rootarea] h5path = "PerturbedEquilibrium/forcing_b_root_area" @@ -75,6 +82,7 @@ extract = "all_complex" label = "forcing b~ (root-area-weighted), n = 1, 2" noise_threshold = 1e-10 order = 51 +class = "physics_converged" [quantities.response_b_rootarea] h5path = "PerturbedEquilibrium/response_b_root_area" @@ -83,6 +91,7 @@ extract = "all_complex" label = "response b~ (root-area-weighted), n = 1, 2" noise_threshold = 1e-8 order = 52 +class = "physics_converged" # Perturbed equilibrium energies and the Σ_n torque [quantities.pe_plasma_energy] @@ -92,6 +101,7 @@ extract = "value" label = "PE plasma energy (sum over n)" noise_threshold = 1e-10 order = 60 +class = "physics_converged" [quantities.pe_vacuum_energy] h5path = "PerturbedEquilibrium/Energies/vacuum_energy" @@ -100,6 +110,7 @@ extract = "value" label = "PE vacuum energy (sum over n)" noise_threshold = 1e-10 order = 61 +class = "physics_converged" [quantities.pe_surface_energy] h5path = "PerturbedEquilibrium/Energies/surface_energy" @@ -108,6 +119,7 @@ extract = "value" label = "PE surface energy (sum over n)" noise_threshold = 1e-10 order = 62 +class = "physics_converged" [quantities.pe_toroidal_torque] h5path = "PerturbedEquilibrium/Energies/toroidal_torque" @@ -116,6 +128,7 @@ extract = "value" label = "PE toroidal torque sum_n -2n Im(Phi_n' Lambda_nn^-1 Phi_n)/4" noise_threshold = 1e-12 order = 63 +class = "physics_converged" # Singular coupling: one row per resonant (surface, n); integer-q surfaces appear once per n [quantities.rational_n] @@ -125,6 +138,7 @@ extract = "all_real" label = "resonant n per row (rows sorted by psi across n)" noise_threshold = 0 order = 70 +class = "physics_converged" [quantities.rational_m] h5path = "PerturbedEquilibrium/SingularCoupling/rational_m" @@ -133,6 +147,7 @@ extract = "all_real" label = "resonant m per row" noise_threshold = 0 order = 71 +class = "physics_converged" [quantities.resonant_area_weighted_field_all] h5path = "PerturbedEquilibrium/SingularCoupling/resonant_area_weighted_field" @@ -141,6 +156,7 @@ extract = "all_complex" label = "resonant area-weighted field b^r per row" noise_threshold = 1e-8 order = 72 +class = "physics_converged" [quantities.island_half_width] h5path = "PerturbedEquilibrium/SingularCoupling/island_half_width" @@ -149,6 +165,7 @@ extract = "all_real" label = "island half-widths per row" noise_threshold = 1e-8 order = 73 +class = "physics_converged" [quantities.chirikov_parameter] h5path = "PerturbedEquilibrium/SingularCoupling/chirikov_parameter" @@ -157,6 +174,7 @@ extract = "all_real" label = "Chirikov parameter per row" noise_threshold = 1e-8 order = 74 +class = "physics_converged" [quantities.delta_prime_all] h5path = "PerturbedEquilibrium/SingularCoupling/Delta_prime" @@ -165,6 +183,7 @@ extract = "all_complex" label = "forcing-driven Delta' per row" noise_threshold = 1e-8 order = 75 +class = "physics_converged" # Runtime [quantities.runtime] @@ -174,3 +193,4 @@ extract = "value" label = "Runtime (s)" noise_threshold = 0.0 order = 999 +class = "diagnostic" diff --git a/regression-harness/cases/diiid_n1.toml b/regression-harness/cases/diiid_n1.toml index 87544cc4c..e115e8651 100644 --- a/regression-harness/cases/diiid_n1.toml +++ b/regression-harness/cases/diiid_n1.toml @@ -349,6 +349,7 @@ extract = "all_real" label = "dominant-coupling singular values" noise_threshold = 1e-8 order = 84 +class = "physics_converged" [quantities.dominant_forcing_overlap] h5path = "PerturbedEquilibrium/SingularCoupling/DominantMode/forcing_overlap" @@ -357,6 +358,7 @@ extract = "abs_first" label = "|forcing overlap with dominant mode|" noise_threshold = 1e-8 order = 85 +class = "physics_converged" # Perturbed equilibrium: energies [quantities.pe_plasma_energy] @@ -480,6 +482,7 @@ extract = "abs_first" label = "|delta_nominal| of coil set 1" noise_threshold = 1e-10 order = 86 +class = "physics_converged" [quantities.ef_shift_sensitivity] h5path = "ErrorFields/CoilSensitivities/DominantMode/shift_sensitivity" @@ -488,6 +491,7 @@ extract = "norm" label = "||ddelta/d(shift)|| over coil sets" noise_threshold = 1e-10 order = 87 +class = "physics_converged" [quantities.ef_tilt_sensitivity] h5path = "ErrorFields/CoilSensitivities/DominantMode/tilt_sensitivity" @@ -496,6 +500,7 @@ extract = "norm" label = "||ddelta/d(tilt)|| over coil sets" noise_threshold = 1e-10 order = 88 +class = "physics_converged" # Runtime [quantities.runtime]