From 445432eeb15298168cbadc5f26c8951677e84023 Mon Sep 17 00:00:00 2001 From: David Hoffman Date: Tue, 6 Oct 2026 16:51:23 -0700 Subject: [PATCH] Consolidate redundant verifier fixture preflight --- ...L-VERIFICATION-PERFORMANCE-001-contract.md | 60 ++++++++++++++++ ...FICATION-PERFORMANCE-001-test-rationale.md | 47 +++++++++++++ .../LOCAL-VERIFICATION-PERFORMANCE-001.md | 68 +++++++++++++++++++ tests/test_verification.py | 13 ++-- 4 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-contract.md create mode 100644 docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-test-rationale.md create mode 100644 docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001.md diff --git a/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-contract.md b/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-contract.md new file mode 100644 index 0000000..0eb075a --- /dev/null +++ b/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-contract.md @@ -0,0 +1,60 @@ +# Local verification performance contract + +Version 1.0. Owner-authorized in the task request on 2026-10-06. + +## Scope and acceptance + +Optimize the complete local `python tools/verification.py full` command on fixed +compute. Required elapsed time is at most 300 seconds; plan for 200 seconds. +The measured baseline already meets both limits. Prefer a small reduction in +redundant test setup over changes to numerical behavior or the gate. + +Preserve the [local verification contract](LOCAL-VERIFICATION-CONTRACT.md), all +existing scientific behaviors, numerical assertions/tolerances, real command +boundaries, clean-install checks, and every quality/audit/build/install/test/report +gate. Require exact 100% measured statements and branches globally and per owned +package, including subprocesses, copied helpers, and never-imported files. +No workers, threads, cores, exclusions, skips, weakened checks, longer deadlines, +remote execution, pushes, pull requests, or publication may be added. + +The authorized test-only slice may consolidate demonstrably redundant fixture +preparation. Keep all existing test cases and assertions unless a separately +reviewed purpose-to-retained-test mapping establishes equivalence. Do not change +runtime code, workflows, dependency versions, coverage/discovery settings, skills, +delivery instructions, or numerical data. No new product behavior is approved. + +| ID | Input/context | Observable outcome | +| --- | --- | --- | +| P1 | Fresh controlled tool caches; complete command | All gates pass; elapsed time at most 300 s, planning target 200 s | +| P2 | Warm controlled tool caches; complete command | All gates pass under the same resource settings; elapsed time at most 300 s, planning target 200 s | + +All existing contract scenarios remain mandatory regression requirements. P1/P2 +add timing conditions; they do not consolidate the existing behavioral scenarios. + +## Measurement and review + +Freeze baseline `e557c7d5822be5d8413d039454782eb7772c9074` and the full command +before edits. Time externally from preparation/launch to final process exit. +Run locally and sequentially with Python 3.13.12 and the unchanged hashed lock, +one test worker, one Black worker, and one numerical-library thread. Use the same +10-core Mac16,12 host with 16 GiB RAM and unchanged CPU allocation. Clear only +declared tool/repository caches for cold runs; do not claim cold OS caches. +Report cold and warm separately. Retain every attempt and failure. Obtain one +cold plus three warm complete candidate measurements and sufficient matched +baseline measurements. Alternate later baseline/candidate warm measurements. +Required cache preparation remains inside the timer. + +A uses `design-tests` for test/fixture consolidation without reading runtime +implementation or history. B uses `review-work` tests mode in a fresh root +session. Commit the accepted checkpoint. Fresh C uses `implement-task` and may +make zero product edits. D uses `review-work` candidate mode in a fresh root +session after complete passing evidence. Disable optional memory and delegation +inside roles. Use source-free warnings/tracebacks for A/B checks and disclose any +exposure. Independence is procedural, not an access-control guarantee. + +Default allowances: two A/B reviews per window; one C repair after initial C. +No dollar cap was specified and reliable dollar metering is unavailable. Record +elapsed preparation, execution, reviews, and failed attempts; naming this task +does not replenish exhausted scientific repair allowances. Complete with local +changes, matched measurements, retained-case/setup rationale, and independent +review, or a specific classified blocker. diff --git a/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-test-rationale.md b/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-test-rationale.md new file mode 100644 index 0000000..931bf9f --- /dev/null +++ b/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001-test-rationale.md @@ -0,0 +1,47 @@ +# LOCAL-VERIFICATION-PERFORMANCE-001 test rationale + +Role A changes only fixture preparation in `tests/test_verification.py`. +The approved P1/P2 timing conditions and all existing regression requirements +remain unchanged. No numerical tests, data, tolerances, or assertions are edited. + +| Consolidated setup and purpose | Retained evidence | +| --- | --- | +| The `verifier` fixture's identical Black stand-in preflight distinguishes broken fixture execution from verifier failures. | `_external_tool_preflight` runs the same real `python -m black fixture-probe` subprocess once per module, with the same environment construction, working-directory convention, 20 s timeout, exit-status assertion, recorded-tool assertion, and record cleanup. All 15 test functions / 48 parameterized cases still request `verifier`, which depends on this preflight. | + +Each case still constructs a new `VerificationCommand` in its own `tmp_path`. +The module preflight uses a separate `tmp_path_factory` directory. No repository, +tool file, environment dictionary, report, call record, or coverage-data path is +shared with a test. Tests that rewrite tools or configuration keep private files. +The constructor and all existing non-fixture helpers remain unchanged. + +All test bodies and parameter declarations remain unchanged. They retain usage +errors, fast/full operation inventories, failure continuation, invalid-report +rejection, report freshness, real lint/docstring/coverage checks, nested coverage, +undecodable output, inert import, and denied process launch. Every subprocess in +those tests remains real and unchanged. The sole reduction is repeated fixture +preflight execution: 48 probes become one, removing 47 redundant launches. One +additional private fixture directory is prepared for that probe. Hook and doctor +fixtures, including their probes and local Git boundaries, remain unchanged. + +Both local affected-suite attempts passed 48 tests, with zero failures, errors, +or skips. External elapsed times were 21.169 s before and 20.594 s after the edit. +The difference is 0.575 s (about 2.7% of this subset). This single sequential pair +is descriptive; it does not establish a reliable speedup or P1/P2 acceptance. +No full benchmark was run. The coordinator retains cold/warm full measurement, +clean-install checks, and exact global/per-package coverage validation. + +The commands used `/private/tmp/dphtools-verification-env/bin/python -m pytest +tests/test_verification.py --tb=no --junitxml=reports/performance-A/-pytest.xml`. +All six specified numerical-library thread settings and `BLACK_NUM_WORKERS` were +`1`. Before execution, the existing `SOURCEFREE` fixture text configured source-free +line caching, exception rendering, and warning formatting in the runner and a +temporary `sitecustomize.py` inherited by Python children. Coverage configuration +remained opaque and unchanged. No runtime source exposure was observed. + +JUnit XML, complete process logs, commands, environment settings, elapsed times, +allowed-input SHA256 revisions, and preservation checks are retained under +`reports/performance-A/`. The baseline and candidate manifests record all seven +permitted input files; the candidate manifest also records this rationale's hash. +The preservation check confirms unchanged syntax trees for all test functions, +parameters, and existing non-fixture helpers/classes, plus byte-identical hook and +delivery test files. Role A did not commit. Independent role B review is next. diff --git a/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001.md b/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001.md new file mode 100644 index 0000000..68835fe --- /dev/null +++ b/docs/tasks/LOCAL-VERIFICATION-PERFORMANCE-001.md @@ -0,0 +1,68 @@ +# Local verification performance + +Version 1.0. The owner explicitly authorized local investigation and optimization +on 2026-10-06. The [contract](LOCAL-VERIFICATION-PERFORMANCE-001-contract.md) +transcribes that request and its retained gates. No second approval is needed for +the authorized fixture optimization. + +## Baseline and evidence + +Baseline: `e557c7d5822be5d8413d039454782eb7772c9074`, unchanged tracked tree. +Canonical command: `/private/tmp/dphtools-verification-env/bin/python tools/verification.py full`. +Fresh locked environment prepared locally with Python 3.13.12 and +`python -m pip install --require-hashes -r requirements-dev.lock`. Environment +provisioning is a command precondition; no required full-command step is moved +outside its timer. Hardware: Mac16,12, 10 physical/logical cores, 16 GiB RAM. +Worker/thread settings and all installed versions are recorded in the external +[measurement directory](/private/tmp/dphtools-verification-measurements). + +Initial cold-tool-cache profile: 133.938941875 s, exit 0, 1,232 passed, 185 warnings, +zero failed/errored/skipped tests, exact 2,094/2,094 statements and 544/544 branches. +The 13 scheduled tool steps plus report validation passed. Tests consumed +107.6 s of external elapsed time; audit 14.9 s; types 7.1 s. JUnit module totals: +verifier tests 40.8 s; hook tests 15.5 s; rolling-ball tests 10.2 s; doctor tests +6.9 s. The baseline already meets 200 s; optimize only justified redundant setup. +Raw reports: `reports/verification/full-zz2e8v_p/`. External monotonic timing, +stdout, stage boundaries, hashes, resource settings and tree-state checks are in +`/private/tmp/dphtools-verification-measurements/baseline-cold.{json,log}`. + +Cold-tool-cache means this arm's controlled XDG/pip/Matplotlib caches and the +checkout's mypy/pytest/build/egg-info/bytecode caches are removed inside the timer. +Shared platform-managed Black/audit HTTP caches and OS caches are not claimed +cold. Warm runs retain each arm's caches. The same environment/interpreter and +dependency versions serve both arms sequentially; each full run freshly builds +and reinstalls its own wheel before tests. No extra worker/CPU allocation is used. + +## Reviewed test preparation + +Fresh root A consolidated the invariant Black stand-in preflight from 48 launches +to one per module. Every test retains a new private command fixture. No test +case, assertion, numerical contract, timeout, hook, runtime file, configuration, +or gate changed. The [purpose mapping](LOCAL-VERIFICATION-PERFORMANCE-001-test-rationale.md) +documents the retained proof and limits. A's 48-case baseline/candidate checks +both passed; that single subset pair is not full-command performance evidence. + +Fresh root B accepted round 1, identifying `tests/test_verification.py` SHA256 +`804eeebd89e7f52ebe879079326e1c490fee241de64a74cc0ead555f018aba95`. +Its fresh 48 verifier and 24 hook cases passed with zero failures/errors/skips. +The subprocess launch observer confirmed one preflight, all 48 cases retained +private fixtures, and preserved test/helper syntax-tree hashes matched. B found +no defect or implementation exposure. Raw evidence: `reports/performance-A/` +and `reports/performance-B/review.md`, `fresh-runs.json`, and +`verification-probe-receipt.json`. The coordinator commits this accepted checkpoint; +fresh C requires no product edits and runs the complete candidate gate. + +Final results will distinguish the initial profile at the candidate worktree path +from the additional cold priming run at the immutable baseline worktree path. +Both are retained. Later warm baseline/candidate runs alternate sequentially. +The timing harness additionally verifies the lock and installed dependency +versions (apart from the distribution under test) against the frozen environment. + +## Current state + +Approvals: owner request; P1/P2 and all inherited regression contracts retained. +Candidate/checkpoint/reviews/check results, final measurements, blockers, next +action, and metrics will be maintained in the originating task conversation and +its linked local evidence after the reviewed checkpoint is committed. This is +the single Current state pointer, prepared before final candidate verification; +the candidate's own hash/results are recorded outside its tracked tree. diff --git a/tests/test_verification.py b/tests/test_verification.py index 3c88a1f..55323f3 100644 --- a/tests/test_verification.py +++ b/tests/test_verification.py @@ -410,9 +410,10 @@ def reports(self): return sorted((self.repo / "reports" / "verification").glob("*/checks.json")) -@pytest.fixture -def verifier(tmp_path): - command = VerificationCommand(tmp_path) +@pytest.fixture(scope="module") +def _external_tool_preflight(tmp_path_factory): + """Probe the invariant stand-in once, separately from each test's private state.""" + command = VerificationCommand(tmp_path_factory.mktemp("verification-tool-preflight")) probe = subprocess.run( [sys.executable, "-m", "black", "fixture-probe"], cwd=command.repo, @@ -426,7 +427,11 @@ def verifier(tmp_path): assert probe.returncode == 0, f"External-tool fixture failed: {_detail(probe)}" assert command.calls()[0]["tool"] == "black" command.record.unlink() - return command + + +@pytest.fixture +def verifier(tmp_path, _external_tool_preflight): + return VerificationCommand(tmp_path) @pytest.mark.parametrize("args", [(), ("invalid",), ("fast", "unexpected")])