From eac50550d5dccfdfc9976497640a1791439b9293 Mon Sep 17 00:00:00 2001 From: PEEKPerformer <24813380+PEEKPerformer@users.noreply.github.com> Date: Thu, 17 Sep 2026 08:02:41 -0400 Subject: [PATCH 1/2] v2.4.1: turn off git auto-maintenance in the archive git runs `git maintenance run --auto --detach` after every commit. Once the loose-object count passes gc.auto, each commit starts its own detached full repack, and nothing stops a second one from starting while the first is still running. An archive's pack is mostly zstd or git-crypt blobs, so each repack holds gigabytes. One ingest tick makes one commit per changed session, which stacked repacks until a 16 GB Mac with a 7.3 GiB archive ran out of memory and had to be force-restarted. Set maintenance.auto=false and gc.auto=0 in the archive's local git config. init.py sets them after `git init`. ingest.py sets them under the ingest lock before the first commit, which migrates existing archives on their next tick. Loose objects now wait for a manual `git repack -d`. The selftest checks both the init path and the migration path. --- CHANGELOG.md | 25 +++++++++++++++++++++++++ holotype/__init__.py | 2 +- holotype/archive.py | 39 +++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- scripts/ingest.py | 6 ++++++ scripts/init.py | 2 ++ scripts/selftest.py | 22 ++++++++++++++++++++++ 7 files changed, 96 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 226d614..0b1beb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ All notable changes to `holotype`. Format adapted from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.4.1] - 2026-09-17 + +Holotype now turns off git's automatic maintenance in the archive. On a large archive it could exhaust the machine's memory. + +### The failure + +git runs `git maintenance run --auto --detach` after every commit. When the loose-object count passes `gc.auto`, that starts a detached full repack (`git repack --cruft` and `git pack-objects --all`). Nothing stops a second repack from starting while the first is still running: each commit starts its own. One ingest tick makes one commit per changed session, so a busy tick starts several repacks at once. + +An archive's pack is mostly zstd or git-crypt blobs, which do not delta or compress. Every repack therefore reads the whole pack and holds gigabytes while it runs. On a real archive (7.3 GiB pack, 16 GB Mac) the repacks stacked up over one tick to 63 `git` processes and about 44 GB of memory. They never finished, the pack was never rewritten, the loose-object count stayed over the threshold, and every later commit started another repack. The machine ran out of memory and had to be force-restarted. + +Reproduced on git 2.54 with a scratch repository: four commits in a row left five concurrent `pack-objects --all` processes. + +### The fix + +- New `holotype.archive.disable_auto_maintenance()` sets `maintenance.auto=false` and `gc.auto=0` in the archive's local `.git/config`. It writes only when a value differs. +- `init.py` calls it right after `git init`. +- `ingest.py` calls it under the ingest lock, before the tick's first commit. Existing archives are migrated on their next tick with no action from you. +- No archive-format change. Nothing tracked in the archive changes. + +Loose objects now accumulate until you pack them. To pack only the loose objects, without rewriting the existing large pack, run `git -C repack -d`. + +### Tests + +The selftest checks that `init.py` sets both keys, and that `ingest.py` restores them on an archive where they are unset. + ## [2.4.0] — 2026-05-28 The archive's "hash chain" is now a real chain. Until this release, "hash chain" named an aspiration: each `manifest.json` carried the SHA-256 of its own transcript, and `verify.py` checked each file against its *own* manifest. That proves per-file content integrity but binds nothing across deposits — a coordinated edit of a transcript *and* its manifest is internally consistent and passed `verify.py` clean. The only thing recording the set and order of deposits was git's commit DAG, which a self-contained `paper_bundle` `.tar.gz` (shipped with no `.git`) doesn't carry. So a reviewer holding a bundle could confirm self-consistency but not that nothing had been inserted, deleted, or substituted. diff --git a/holotype/__init__.py b/holotype/__init__.py index d586123..61e18c2 100644 --- a/holotype/__init__.py +++ b/holotype/__init__.py @@ -1,3 +1,3 @@ """holotype: forensic-grade archival of agent-CLI sessions for scientific reproducibility.""" -__version__ = "2.4.0" +__version__ = "2.4.1" diff --git a/holotype/archive.py b/holotype/archive.py index 6631a5f..0286f75 100644 --- a/holotype/archive.py +++ b/holotype/archive.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import subprocess from pathlib import Path from typing import Iterator @@ -62,3 +63,41 @@ def resolve_session_by_prefix(archive: Path, prefix: str) -> Path | None: sys.stderr.write(f" {m.name}\n") return None return matches[0] + + +# git runs `git maintenance run --auto --detach` after every commit. Once +# the loose-object count crosses gc.auto, each commit starts its own +# detached full repack (`pack-objects --all`), and nothing stops a second +# one from starting while the first is still running. An archive's pack +# is mostly incompressible blobs (zstd or git-crypt), so each repack holds +# gigabytes and runs for a long time. One ingest tick makes one commit +# per changed session, which is enough to stack repacks until the machine +# runs out of memory. Holotype turns automatic maintenance off in the +# archive; repack by hand with `git repack -d` when loose objects pile up. +AUTO_MAINTENANCE_OFF = ( + ("maintenance.auto", "false"), + ("gc.auto", "0"), +) + + +def disable_auto_maintenance(archive: Path) -> None: + """Turn off git's automatic background maintenance for ``archive``. + + Writes to the archive's local ``.git/config`` only, and only when a + value differs, so calling this every ingest tick is cheap. Failures + are ignored: a missing setting costs memory, not correctness. + """ + for key, value in AUTO_MAINTENANCE_OFF: + try: + current = subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--get", key], + capture_output=True, text=True, timeout=10, + ) + if current.stdout.strip() == value: + continue + subprocess.run( + ["git", "-C", str(archive), "config", "--local", key, value], + capture_output=True, text=True, timeout=10, + ) + except (subprocess.SubprocessError, OSError): + pass diff --git a/pyproject.toml b/pyproject.toml index 650e6ee..8771f9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "holotype" -version = "2.4.0" +version = "2.4.1" description = "Forensic-grade archival of LLM-driven agent CLI sessions for scientific reproducibility" readme = "README.md" license = { file = "LICENSE" } diff --git a/scripts/ingest.py b/scripts/ingest.py index e65da3c..437cdac 100644 --- a/scripts/ingest.py +++ b/scripts/ingest.py @@ -42,6 +42,7 @@ sys.path.insert(0, str(REPO_ROOT)) from holotype import __version__ +from holotype.archive import disable_auto_maintenance from holotype.chunking import bin_pack_paths, dir_size_bytes from holotype.compression import compress_bytes, find_transcript, transcript_filename from holotype.env import claude_code_version, git_state_for_path, platform_info @@ -902,6 +903,11 @@ def main(argv: list[str] | None = None) -> int: print("holotype: another ingest is running, exiting") return 0 + # Before the first commit of this tick, so no commit can start a + # detached repack. Also migrates archives created before this setting. + if not args.dry_run: + disable_auto_maintenance(archive) + try: counts: dict[str, int] = {"new": 0, "updated": 0, "skipped-live": 0, "skipped-unchanged": 0, "skipped-empty": 0} diff --git a/scripts/init.py b/scripts/init.py index 991e126..c08d6dd 100644 --- a/scripts/init.py +++ b/scripts/init.py @@ -35,6 +35,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from holotype import ledger # noqa: E402 +from holotype.archive import disable_auto_maintenance # noqa: E402 ARCHIVE_FORMAT_VERSION = 1 HOLOTYPE_SUBDIR = ".holotype" @@ -525,6 +526,7 @@ def init_archive(args: argparse.Namespace) -> int: is_new_repo = not (archive / ".git").exists() if is_new_repo: run(["git", "init", "-b", "main"], cwd=archive) + disable_auto_maintenance(archive) if args.sign_commits: # Probe for a signing key. Don't fail init outright — the user diff --git a/scripts/selftest.py b/scripts/selftest.py index c4f586d..9f7503e 100644 --- a/scripts/selftest.py +++ b/scripts/selftest.py @@ -171,6 +171,20 @@ def main(argv: list[str] | None = None) -> int: expect((archive / "README.md").exists(), "archive README.md missing") expect((archive / ".git").is_dir(), "archive is not a git repo") + step("init.py turns off git auto-maintenance in the archive") + for key, want in (("maintenance.auto", "false"), ("gc.auto", "0")): + got = subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--get", key], + capture_output=True, text=True, + ).stdout.strip() + expect(got == want, f"archive {key}={got!r}, expected {want!r}") + # Unset again so the first ingest below has to migrate an + # archive created before this setting existed. + subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--unset", key], + check=True, + ) + step("materialize synthetic source") source = materialize_source(tmp) @@ -186,6 +200,14 @@ def main(argv: list[str] | None = None) -> int: expect(result.returncode == 0, f"ingest.py returncode={result.returncode}: {result.stderr}") expect("new=3" in result.stdout, f"expected new=3 in output:\n{result.stdout}") + step("ingest turns off git auto-maintenance on an older archive") + for key, want in (("maintenance.auto", "false"), ("gc.auto", "0")): + got = subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--get", key], + capture_output=True, text=True, + ).stdout.strip() + expect(got == want, f"archive {key}={got!r} after ingest, expected {want!r}") + sessions_dir = archive / "sessions" deposits = sorted(p for p in sessions_dir.rglob("transcript.jsonl")) expect(len(deposits) == 3, f"expected 3 deposits (2 top-level + 1 subagent), got {len(deposits)}") From f5b8b6fac66826b322d009e4b1f885cd5c1a5d15 Mon Sep 17 00:00:00 2001 From: PEEKPerformer <24813380+PEEKPerformer@users.noreply.github.com> Date: Thu, 17 Sep 2026 08:16:21 -0400 Subject: [PATCH 2/2] v2.4.1: stop reporting stale compressed hashes as damage On a real archive 1,832 compressed deposits failed the sweep fsck, yet every one decompressed to its canonical sha256. The v2.3.0 pure migration copied sha256_compressed from the prior manifest instead of hashing the file, so stale values survived, and the fsck compares only the compressed hash. About 40 false failures per sweep slice buried any real one. Hash the file during a pure migration. In the fsck, fall back to the canonical hash when the compressed one disagrees and count a match as stale, not failed. Add scripts/repair_compressed_hashes.py to rewrite the stale values in one commit, touching only that field and only after the canonical hash matches. --- CHANGELOG.md | 12 +- scripts/ingest.py | 48 ++++++-- scripts/repair_compressed_hashes.py | 174 ++++++++++++++++++++++++++++ scripts/selftest.py | 49 ++++++++ 4 files changed, 274 insertions(+), 9 deletions(-) create mode 100644 scripts/repair_compressed_hashes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b1beb2..29fd32d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,9 +23,19 @@ Reproduced on git 2.54 with a scratch repository: four commits in a row left fiv Loose objects now accumulate until you pack them. To pack only the loose objects, without rewriting the existing large pack, run `git -C repack -d`. +### Stale compressed hashes no longer read as integrity failures + +On a real archive, 1,832 of 5,315 compressed deposits failed the rolling-sweep fsck with `ARCHIVE INTEGRITY FAILURE` ("stored bytes do not match manifest hash"). Every one of them decompressed to exactly the canonical `sha256`. The content was intact; only `sha256_compressed` was wrong. All 1,832 were deposited on the same day by v2.1.0 and migrated by v2.3.0. + +The v2.3.0 pure-migration path copied `sha256_compressed` from the prior manifest instead of hashing the file, so a stale value survived the migration. The sweep checks only the compressed hash, so each sweep slice reported about 40 false failures, and the real signal was lost in them. + +- The pure-migration path now hashes the `.zst` file on disk. +- The fsck decompresses a deposit whose compressed hash disagrees. If the canonical `sha256` matches, it counts the deposit as `stale-hash`, not as an integrity failure. Real damage is still reported as before. +- New `scripts/repair_compressed_hashes.py` rewrites stale values in one `repair:` commit. It changes only the `sha256_compressed` string, and only after the canonical hash matches, so transcripts and the ledger are untouched. `--dry-run` reports without writing. Damaged deposits are listed and never rewritten. + ### Tests -The selftest checks that `init.py` sets both keys, and that `ingest.py` restores them on an archive where they are unset. +The selftest checks that `init.py` sets both keys, and that `ingest.py` restores them on an archive where they are unset. A new step covers stale compressed hashes: the migration rehashes, the fsck reports stale instead of failed, the repair commits, and real damage is still caught. ## [2.4.0] — 2026-05-28 diff --git a/scripts/ingest.py b/scripts/ingest.py index 437cdac..af4296e 100644 --- a/scripts/ingest.py +++ b/scripts/ingest.py @@ -44,7 +44,13 @@ from holotype import __version__ from holotype.archive import disable_auto_maintenance from holotype.chunking import bin_pack_paths, dir_size_bytes -from holotype.compression import compress_bytes, find_transcript, transcript_filename +from holotype.compression import ( + compress_bytes, + decompress_bytes, + find_transcript, + transcript_filename, + zstd_available, +) from holotype.env import claude_code_version, git_state_for_path, platform_info from holotype.hashing import sha256_bytes from holotype.index import ( @@ -265,15 +271,22 @@ def needs_processing( def verify_deposit_integrity( - archive: Path, candidate: DepositCandidate + archive: Path, + candidate: DepositCandidate, + stale_out: list[str] | None = None, ) -> str | None: """fsck one deposited transcript against its manifest hash. Hashes the on-disk deposit bytes as they sit (compressed bytes vs - ``sha256_compressed`` when zstd, raw bytes vs ``sha256`` otherwise) — - pure bit-rot detection that needs no zstd and never false-alarms. + ``sha256_compressed`` when zstd, raw bytes vs ``sha256`` otherwise). Returns an error string on mismatch/missing, else None. Run on the rolling-sweep slice so the whole archive is re-checked every ~24h. + + When only the compressed hash disagrees, the file is decompressed and + checked against the canonical ``sha256``. If that matches, the content + is intact and only the recorded compressed hash is stale: the session + is appended to ``stale_out`` instead of being reported as a failure. + ``scripts/repair_compressed_hashes.py`` rewrites the stale values. """ m = existing_manifest(archive, candidate) if not m: @@ -291,6 +304,15 @@ def verify_deposit_integrity( except OSError as e: return f"{candidate.archive_subpath}: deposited transcript unreadable ({e})" if sha256_bytes(raw) != expected: + if is_compressed and m.get("sha256") and zstd_available(): + try: + canonical = sha256_bytes(decompress_bytes(raw)) + except Exception: + canonical = None + if canonical == m.get("sha256"): + if stale_out is not None: + stale_out.append(candidate.archive_subpath) + return None return ( f"{candidate.archive_subpath}: ARCHIVE INTEGRITY FAILURE — " f"stored bytes do not match manifest hash" @@ -398,9 +420,11 @@ def deposit_one( if is_pure_migration and dest_transcript.exists(): if compression == "zstd": - sha_compressed = prior_manifest.get("sha256_compressed") or sha256_bytes( - dest_transcript.read_bytes() - ) + # Hash the file as it sits instead of copying the prior + # manifest's value. A copied value can be stale: the v2.3.0 + # migration carried 1,832 stale compressed hashes forward on a + # real archive, and each one read as an integrity failure. + sha_compressed = sha256_bytes(dest_transcript.read_bytes()) else: sha_compressed = None else: @@ -1250,9 +1274,10 @@ def main(argv: list[str] | None = None) -> int: # Failures are reported loudly and sent to stderr — they indicate # the deposited artifact no longer matches what was recorded. integrity_failures: list[str] = [] + stale_compressed: list[str] = [] if not args.dry_run: for _cls, cand in sweep_candidates: - err = verify_deposit_integrity(archive, cand) + err = verify_deposit_integrity(archive, cand, stale_compressed) if err: integrity_failures.append(err) advance_sweep_cursor(archive, sweep_bucket) @@ -1263,6 +1288,12 @@ def main(argv: list[str] | None = None) -> int: ) for f in integrity_failures: sys.stderr.write(f" {f}\n") + if stale_compressed and not args.quiet: + print( + f"holotype: {len(stale_compressed)} deposit(s) in this sweep have a " + f"stale sha256_compressed (content verified intact). Fix with " + f"`python scripts/repair_compressed_hashes.py`." + ) any_changes = counts["new"] + counts["updated"] > 0 if not args.quiet or any_changes or integrity_failures: @@ -1273,6 +1304,7 @@ def main(argv: list[str] | None = None) -> int: f" empty={counts['skipped-empty']}" f" swept={len(sweep_candidates)}" + (f" INTEGRITY-FAIL={len(integrity_failures)}" if integrity_failures else "") + + (f" stale-hash={len(stale_compressed)}" if stale_compressed else "") ) if committed and not args.quiet: print("\n".join(committed)) diff --git a/scripts/repair_compressed_hashes.py b/scripts/repair_compressed_hashes.py new file mode 100644 index 0000000..63e1ae7 --- /dev/null +++ b/scripts/repair_compressed_hashes.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Rewrite stale ``sha256_compressed`` values in deposit manifests. + +A compressed deposit records two hashes: ``sha256`` of the canonical +(uncompressed) transcript, which is what gets cited and chained, and +``sha256_compressed`` of the ``.zst`` file as it sits on disk, which lets a +reviewer without zstd check the file directly. + +Before v2.4.1, a pure manifest-version migration copied +``sha256_compressed`` from the prior manifest instead of hashing the file. +When the prior value was already stale, the migration carried it forward. +The content is intact (the canonical hash still matches), but the ingest +sweep and the no-zstd track of ``verify.py`` both report the deposit as an +integrity failure. + +This script finds every compressed deposit whose file hash differs from +``sha256_compressed``, decompresses it, and checks the result against +``sha256``. Only when the canonical hash matches does it rewrite +``sha256_compressed``. A deposit whose canonical hash does NOT match is real +damage: it is reported and left alone. + +The rewrite changes one hex string in each manifest and nothing else, so the +hash-chain ledger (which links canonical ``sha256`` values only) is +unaffected. All repaired manifests go into one ``repair:`` commit. + +Usage: + python scripts/repair_compressed_hashes.py --dry-run # report only + python scripts/repair_compressed_hashes.py # repair + commit + +Exit codes: + 0 nothing stale, or all stale hashes repaired + 1 at least one deposit has real content damage (never rewritten) + 2 another ingest holds the archive lock, or zstd is missing +""" + +from __future__ import annotations + +import argparse +import fcntl +import json +import os +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from holotype.archive import iter_all_sessions # noqa: E402 +from holotype.compression import ( # noqa: E402 + COMPRESSED_TRANSCRIPT, + decompress_bytes, + zstd_available, +) +from holotype.hashing import sha256_bytes # noqa: E402 + + +def find_archive(explicit: Path | None) -> Path: + if explicit: + return explicit.expanduser().resolve() + env = os.environ.get("HOLOTYPE_ARCHIVE") + if env: + return Path(env).expanduser().resolve() + pointer = Path.home() / ".config" / "holotype" / "archive-path" + if pointer.exists(): + return Path(pointer.read_text().strip()).expanduser().resolve() + return (Path.home() / "Documents" / "holotype-archive").resolve() + + +def run_git(archive: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-C", str(archive), *args], capture_output=True, text=True + ) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--archive", type=Path, default=None) + p.add_argument("--dry-run", action="store_true", + help="Report stale hashes without rewriting anything.") + args = p.parse_args(argv) + + if not zstd_available(): + print("repair: zstd not found on PATH; cannot check canonical hashes.", + file=sys.stderr) + return 2 + + archive = find_archive(args.archive) + lock_path = archive / ".holotype" / ".lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + lock = lock_path.open("w") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("repair: an ingest is running; try again when it finishes.", + file=sys.stderr) + return 2 + + try: + stale: list[tuple[Path, str, str]] = [] + damaged: list[Path] = [] + checked = 0 + for sess_dir, manifest in iter_all_sessions(archive): + zst = sess_dir / COMPRESSED_TRANSCRIPT + recorded = manifest.get("sha256_compressed") + if not zst.exists() or not recorded: + continue + checked += 1 + raw = zst.read_bytes() + actual = sha256_bytes(raw) + if actual == recorded: + continue + try: + canonical = sha256_bytes(decompress_bytes(raw)) + except Exception: + canonical = None + if canonical and canonical == manifest.get("sha256"): + stale.append((sess_dir, recorded, actual)) + else: + damaged.append(sess_dir) + + rel = lambda d: d.relative_to(archive) # noqa: E731 + print(f"repair: checked {checked} compressed deposit(s): " + f"{len(stale)} stale hash(es), {len(damaged)} damaged") + for d in damaged: + print(f" DAMAGED (not rewritten): {rel(d)}", file=sys.stderr) + + if args.dry_run or not stale: + return 1 if damaged else 0 + + for sess_dir, old, new in stale: + path = sess_dir / "manifest.json" + text = path.read_text() + if text.count(old) != 1: + print(f" skipped (hash not found exactly once): {rel(sess_dir)}", + file=sys.stderr) + continue + tmp = path.with_suffix(".json.partial") + tmp.write_text(text.replace(old, new)) + os.replace(tmp, path) + + paths = [f"{rel(d)}/manifest.json" for d, _, _ in stale] + for i in range(0, len(paths), 500): + run_git(archive, "add", *paths[i:i + 500]) + if run_git(archive, "diff", "--cached", "--quiet").returncode != 0: + config_path = archive / ".holotype" / "config.json" + try: + config = json.loads(config_path.read_text()) + except (OSError, json.JSONDecodeError): + config = {} + sign = bool((config.get("deposit") or {}).get("sign_commits")) + msg = ( + f"repair: rewrite {len(stale)} stale sha256_compressed value(s)\n\n" + "Each manifest's sha256_compressed no longer matched its .zst\n" + "file. The decompressed content still matches the canonical\n" + "sha256, so only the compressed hash was rewritten. Transcripts\n" + "and the hash-chain ledger are unchanged." + ) + commit_args = ["commit", "-m", msg] + if sign: + commit_args.insert(1, "-S") + out = run_git(archive, *commit_args) + if out.returncode != 0: + print(f"repair: commit failed: {out.stderr.strip()}", file=sys.stderr) + return 2 + print(f"repair: committed {len(stale)} manifest(s)") + return 1 if damaged else 0 + finally: + fcntl.flock(lock, fcntl.LOCK_UN) + lock.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/selftest.py b/scripts/selftest.py index 9f7503e..a467826 100644 --- a/scripts/selftest.py +++ b/scripts/selftest.py @@ -1581,6 +1581,55 @@ def main(argv: list[str] | None = None) -> int: expect(res.returncode == 0, f"verify failed after pure migration: {res.stderr or res.stdout}") + step("stale sha256_compressed: migration rehashes, fsck and repair tell it from damage") + # A stale recorded compressed hash (content intact) must not + # survive a migration: the migration hashes the file on disk. + zc_obj = json.loads(zc_manifest.read_text()) + zc_obj["manifest_version"] = 3 + zc_obj["sha256_compressed"] = "0" * 64 + zc_manifest.write_text(json.dumps(zc_obj, indent=2, sort_keys=True) + "\n") + res = run_script( + REPO_ROOT / "scripts" / "ingest.py", "--archive", str(zc_archive), + "--source", str(zc_src), "--source-name", "claude-code", "--workers", "1", + ) + expect(res.returncode == 0, f"zc stale-hash migration failed: {res.stderr}") + post = json.loads(zc_manifest.read_text()) + expect(post.get("sha256_compressed") == hashlib.sha256(zc_zst.read_bytes()).hexdigest(), + "migration carried a stale sha256_compressed forward") + + # The fsck reports a stale hash as stale, not as an integrity failure. + zc_obj = json.loads(zc_manifest.read_text()) + real_hash = zc_obj["sha256_compressed"] + zc_obj["sha256_compressed"] = "0" * 64 + zc_manifest.write_text(json.dumps(zc_obj, indent=2, sort_keys=True) + "\n") + # Commit the stale value, as it would be in a real archive. + subprocess.run(["git", "-C", str(zc_archive), "commit", "-qam", "test: stale hash"], + check=True) + stale: list[str] = [] + fsck_err = _ig.verify_deposit_integrity(zc_archive, zc_cand, stale) + expect(fsck_err is None and stale == [zc_cand.archive_subpath], + f"stale compressed hash misreported (err={fsck_err!r}, stale={stale})") + + # The repair script rewrites it and commits; verify is clean after. + res = run_script(REPO_ROOT / "scripts" / "repair_compressed_hashes.py", + "--archive", str(zc_archive)) + expect(res.returncode == 0, f"repair failed: {res.stderr or res.stdout}") + expect(json.loads(zc_manifest.read_text())["sha256_compressed"] == real_hash, + "repair did not restore the real sha256_compressed") + log = subprocess.run(["git", "-C", str(zc_archive), "log", "-1", "--format=%s"], + capture_output=True, text=True).stdout + expect(log.startswith("repair:"), f"repair did not commit (HEAD: {log!r})") + + # Real damage is still an integrity failure and is never rewritten. + zc_zst.write_bytes(b"not zstd at all") + stale = [] + fsck_err = _ig.verify_deposit_integrity(zc_archive, zc_cand, stale) + expect(fsck_err is not None and "INTEGRITY" in fsck_err and not stale, + f"damaged deposit not reported as failure (err={fsck_err!r})") + res = run_script(REPO_ROOT / "scripts" / "repair_compressed_hashes.py", + "--archive", str(zc_archive)) + expect(res.returncode == 1, f"repair should exit 1 on damage, got {res.returncode}") + print("\nALL CHECKS PASSED") success = True return 0