diff --git a/CHANGELOG.md b/CHANGELOG.md index 226d614..29fd32d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,41 @@ All notable changes to `holotype`. Format adapted from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.4.1] - 2026-09-17 + +Holotype now turns off git's automatic maintenance in the archive. On a large archive it could exhaust the machine's memory. + +### The failure + +git runs `git maintenance run --auto --detach` after every commit. When the loose-object count passes `gc.auto`, that starts a detached full repack (`git repack --cruft` and `git pack-objects --all`). Nothing stops a second repack from starting while the first is still running: each commit starts its own. One ingest tick makes one commit per changed session, so a busy tick starts several repacks at once. + +An archive's pack is mostly zstd or git-crypt blobs, which do not delta or compress. Every repack therefore reads the whole pack and holds gigabytes while it runs. On a real archive (7.3 GiB pack, 16 GB Mac) the repacks stacked up over one tick to 63 `git` processes and about 44 GB of memory. They never finished, the pack was never rewritten, the loose-object count stayed over the threshold, and every later commit started another repack. The machine ran out of memory and had to be force-restarted. + +Reproduced on git 2.54 with a scratch repository: four commits in a row left five concurrent `pack-objects --all` processes. + +### The fix + +- New `holotype.archive.disable_auto_maintenance()` sets `maintenance.auto=false` and `gc.auto=0` in the archive's local `.git/config`. It writes only when a value differs. +- `init.py` calls it right after `git init`. +- `ingest.py` calls it under the ingest lock, before the tick's first commit. Existing archives are migrated on their next tick with no action from you. +- No archive-format change. Nothing tracked in the archive changes. + +Loose objects now accumulate until you pack them. To pack only the loose objects, without rewriting the existing large pack, run `git -C repack -d`. + +### Stale compressed hashes no longer read as integrity failures + +On a real archive, 1,832 of 5,315 compressed deposits failed the rolling-sweep fsck with `ARCHIVE INTEGRITY FAILURE` ("stored bytes do not match manifest hash"). Every one of them decompressed to exactly the canonical `sha256`. The content was intact; only `sha256_compressed` was wrong. All 1,832 were deposited on the same day by v2.1.0 and migrated by v2.3.0. + +The v2.3.0 pure-migration path copied `sha256_compressed` from the prior manifest instead of hashing the file, so a stale value survived the migration. The sweep checks only the compressed hash, so each sweep slice reported about 40 false failures, and the real signal was lost in them. + +- The pure-migration path now hashes the `.zst` file on disk. +- The fsck decompresses a deposit whose compressed hash disagrees. If the canonical `sha256` matches, it counts the deposit as `stale-hash`, not as an integrity failure. Real damage is still reported as before. +- New `scripts/repair_compressed_hashes.py` rewrites stale values in one `repair:` commit. It changes only the `sha256_compressed` string, and only after the canonical hash matches, so transcripts and the ledger are untouched. `--dry-run` reports without writing. Damaged deposits are listed and never rewritten. + +### Tests + +The selftest checks that `init.py` sets both keys, and that `ingest.py` restores them on an archive where they are unset. A new step covers stale compressed hashes: the migration rehashes, the fsck reports stale instead of failed, the repair commits, and real damage is still caught. + ## [2.4.0] — 2026-05-28 The archive's "hash chain" is now a real chain. Until this release, "hash chain" named an aspiration: each `manifest.json` carried the SHA-256 of its own transcript, and `verify.py` checked each file against its *own* manifest. That proves per-file content integrity but binds nothing across deposits — a coordinated edit of a transcript *and* its manifest is internally consistent and passed `verify.py` clean. The only thing recording the set and order of deposits was git's commit DAG, which a self-contained `paper_bundle` `.tar.gz` (shipped with no `.git`) doesn't carry. So a reviewer holding a bundle could confirm self-consistency but not that nothing had been inserted, deleted, or substituted. diff --git a/holotype/__init__.py b/holotype/__init__.py index d586123..61e18c2 100644 --- a/holotype/__init__.py +++ b/holotype/__init__.py @@ -1,3 +1,3 @@ """holotype: forensic-grade archival of agent-CLI sessions for scientific reproducibility.""" -__version__ = "2.4.0" +__version__ = "2.4.1" diff --git a/holotype/archive.py b/holotype/archive.py index 6631a5f..0286f75 100644 --- a/holotype/archive.py +++ b/holotype/archive.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import subprocess from pathlib import Path from typing import Iterator @@ -62,3 +63,41 @@ def resolve_session_by_prefix(archive: Path, prefix: str) -> Path | None: sys.stderr.write(f" {m.name}\n") return None return matches[0] + + +# git runs `git maintenance run --auto --detach` after every commit. Once +# the loose-object count crosses gc.auto, each commit starts its own +# detached full repack (`pack-objects --all`), and nothing stops a second +# one from starting while the first is still running. An archive's pack +# is mostly incompressible blobs (zstd or git-crypt), so each repack holds +# gigabytes and runs for a long time. One ingest tick makes one commit +# per changed session, which is enough to stack repacks until the machine +# runs out of memory. Holotype turns automatic maintenance off in the +# archive; repack by hand with `git repack -d` when loose objects pile up. +AUTO_MAINTENANCE_OFF = ( + ("maintenance.auto", "false"), + ("gc.auto", "0"), +) + + +def disable_auto_maintenance(archive: Path) -> None: + """Turn off git's automatic background maintenance for ``archive``. + + Writes to the archive's local ``.git/config`` only, and only when a + value differs, so calling this every ingest tick is cheap. Failures + are ignored: a missing setting costs memory, not correctness. + """ + for key, value in AUTO_MAINTENANCE_OFF: + try: + current = subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--get", key], + capture_output=True, text=True, timeout=10, + ) + if current.stdout.strip() == value: + continue + subprocess.run( + ["git", "-C", str(archive), "config", "--local", key, value], + capture_output=True, text=True, timeout=10, + ) + except (subprocess.SubprocessError, OSError): + pass diff --git a/pyproject.toml b/pyproject.toml index 650e6ee..8771f9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "holotype" -version = "2.4.0" +version = "2.4.1" description = "Forensic-grade archival of LLM-driven agent CLI sessions for scientific reproducibility" readme = "README.md" license = { file = "LICENSE" } diff --git a/scripts/ingest.py b/scripts/ingest.py index e65da3c..af4296e 100644 --- a/scripts/ingest.py +++ b/scripts/ingest.py @@ -42,8 +42,15 @@ sys.path.insert(0, str(REPO_ROOT)) from holotype import __version__ +from holotype.archive import disable_auto_maintenance from holotype.chunking import bin_pack_paths, dir_size_bytes -from holotype.compression import compress_bytes, find_transcript, transcript_filename +from holotype.compression import ( + compress_bytes, + decompress_bytes, + find_transcript, + transcript_filename, + zstd_available, +) from holotype.env import claude_code_version, git_state_for_path, platform_info from holotype.hashing import sha256_bytes from holotype.index import ( @@ -264,15 +271,22 @@ def needs_processing( def verify_deposit_integrity( - archive: Path, candidate: DepositCandidate + archive: Path, + candidate: DepositCandidate, + stale_out: list[str] | None = None, ) -> str | None: """fsck one deposited transcript against its manifest hash. Hashes the on-disk deposit bytes as they sit (compressed bytes vs - ``sha256_compressed`` when zstd, raw bytes vs ``sha256`` otherwise) — - pure bit-rot detection that needs no zstd and never false-alarms. + ``sha256_compressed`` when zstd, raw bytes vs ``sha256`` otherwise). Returns an error string on mismatch/missing, else None. Run on the rolling-sweep slice so the whole archive is re-checked every ~24h. + + When only the compressed hash disagrees, the file is decompressed and + checked against the canonical ``sha256``. If that matches, the content + is intact and only the recorded compressed hash is stale: the session + is appended to ``stale_out`` instead of being reported as a failure. + ``scripts/repair_compressed_hashes.py`` rewrites the stale values. """ m = existing_manifest(archive, candidate) if not m: @@ -290,6 +304,15 @@ def verify_deposit_integrity( except OSError as e: return f"{candidate.archive_subpath}: deposited transcript unreadable ({e})" if sha256_bytes(raw) != expected: + if is_compressed and m.get("sha256") and zstd_available(): + try: + canonical = sha256_bytes(decompress_bytes(raw)) + except Exception: + canonical = None + if canonical == m.get("sha256"): + if stale_out is not None: + stale_out.append(candidate.archive_subpath) + return None return ( f"{candidate.archive_subpath}: ARCHIVE INTEGRITY FAILURE — " f"stored bytes do not match manifest hash" @@ -397,9 +420,11 @@ def deposit_one( if is_pure_migration and dest_transcript.exists(): if compression == "zstd": - sha_compressed = prior_manifest.get("sha256_compressed") or sha256_bytes( - dest_transcript.read_bytes() - ) + # Hash the file as it sits instead of copying the prior + # manifest's value. A copied value can be stale: the v2.3.0 + # migration carried 1,832 stale compressed hashes forward on a + # real archive, and each one read as an integrity failure. + sha_compressed = sha256_bytes(dest_transcript.read_bytes()) else: sha_compressed = None else: @@ -902,6 +927,11 @@ def main(argv: list[str] | None = None) -> int: print("holotype: another ingest is running, exiting") return 0 + # Before the first commit of this tick, so no commit can start a + # detached repack. Also migrates archives created before this setting. + if not args.dry_run: + disable_auto_maintenance(archive) + try: counts: dict[str, int] = {"new": 0, "updated": 0, "skipped-live": 0, "skipped-unchanged": 0, "skipped-empty": 0} @@ -1244,9 +1274,10 @@ def main(argv: list[str] | None = None) -> int: # Failures are reported loudly and sent to stderr — they indicate # the deposited artifact no longer matches what was recorded. integrity_failures: list[str] = [] + stale_compressed: list[str] = [] if not args.dry_run: for _cls, cand in sweep_candidates: - err = verify_deposit_integrity(archive, cand) + err = verify_deposit_integrity(archive, cand, stale_compressed) if err: integrity_failures.append(err) advance_sweep_cursor(archive, sweep_bucket) @@ -1257,6 +1288,12 @@ def main(argv: list[str] | None = None) -> int: ) for f in integrity_failures: sys.stderr.write(f" {f}\n") + if stale_compressed and not args.quiet: + print( + f"holotype: {len(stale_compressed)} deposit(s) in this sweep have a " + f"stale sha256_compressed (content verified intact). Fix with " + f"`python scripts/repair_compressed_hashes.py`." + ) any_changes = counts["new"] + counts["updated"] > 0 if not args.quiet or any_changes or integrity_failures: @@ -1267,6 +1304,7 @@ def main(argv: list[str] | None = None) -> int: f" empty={counts['skipped-empty']}" f" swept={len(sweep_candidates)}" + (f" INTEGRITY-FAIL={len(integrity_failures)}" if integrity_failures else "") + + (f" stale-hash={len(stale_compressed)}" if stale_compressed else "") ) if committed and not args.quiet: print("\n".join(committed)) diff --git a/scripts/init.py b/scripts/init.py index 991e126..c08d6dd 100644 --- a/scripts/init.py +++ b/scripts/init.py @@ -35,6 +35,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from holotype import ledger # noqa: E402 +from holotype.archive import disable_auto_maintenance # noqa: E402 ARCHIVE_FORMAT_VERSION = 1 HOLOTYPE_SUBDIR = ".holotype" @@ -525,6 +526,7 @@ def init_archive(args: argparse.Namespace) -> int: is_new_repo = not (archive / ".git").exists() if is_new_repo: run(["git", "init", "-b", "main"], cwd=archive) + disable_auto_maintenance(archive) if args.sign_commits: # Probe for a signing key. Don't fail init outright — the user diff --git a/scripts/repair_compressed_hashes.py b/scripts/repair_compressed_hashes.py new file mode 100644 index 0000000..63e1ae7 --- /dev/null +++ b/scripts/repair_compressed_hashes.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Rewrite stale ``sha256_compressed`` values in deposit manifests. + +A compressed deposit records two hashes: ``sha256`` of the canonical +(uncompressed) transcript, which is what gets cited and chained, and +``sha256_compressed`` of the ``.zst`` file as it sits on disk, which lets a +reviewer without zstd check the file directly. + +Before v2.4.1, a pure manifest-version migration copied +``sha256_compressed`` from the prior manifest instead of hashing the file. +When the prior value was already stale, the migration carried it forward. +The content is intact (the canonical hash still matches), but the ingest +sweep and the no-zstd track of ``verify.py`` both report the deposit as an +integrity failure. + +This script finds every compressed deposit whose file hash differs from +``sha256_compressed``, decompresses it, and checks the result against +``sha256``. Only when the canonical hash matches does it rewrite +``sha256_compressed``. A deposit whose canonical hash does NOT match is real +damage: it is reported and left alone. + +The rewrite changes one hex string in each manifest and nothing else, so the +hash-chain ledger (which links canonical ``sha256`` values only) is +unaffected. All repaired manifests go into one ``repair:`` commit. + +Usage: + python scripts/repair_compressed_hashes.py --dry-run # report only + python scripts/repair_compressed_hashes.py # repair + commit + +Exit codes: + 0 nothing stale, or all stale hashes repaired + 1 at least one deposit has real content damage (never rewritten) + 2 another ingest holds the archive lock, or zstd is missing +""" + +from __future__ import annotations + +import argparse +import fcntl +import json +import os +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from holotype.archive import iter_all_sessions # noqa: E402 +from holotype.compression import ( # noqa: E402 + COMPRESSED_TRANSCRIPT, + decompress_bytes, + zstd_available, +) +from holotype.hashing import sha256_bytes # noqa: E402 + + +def find_archive(explicit: Path | None) -> Path: + if explicit: + return explicit.expanduser().resolve() + env = os.environ.get("HOLOTYPE_ARCHIVE") + if env: + return Path(env).expanduser().resolve() + pointer = Path.home() / ".config" / "holotype" / "archive-path" + if pointer.exists(): + return Path(pointer.read_text().strip()).expanduser().resolve() + return (Path.home() / "Documents" / "holotype-archive").resolve() + + +def run_git(archive: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-C", str(archive), *args], capture_output=True, text=True + ) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--archive", type=Path, default=None) + p.add_argument("--dry-run", action="store_true", + help="Report stale hashes without rewriting anything.") + args = p.parse_args(argv) + + if not zstd_available(): + print("repair: zstd not found on PATH; cannot check canonical hashes.", + file=sys.stderr) + return 2 + + archive = find_archive(args.archive) + lock_path = archive / ".holotype" / ".lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + lock = lock_path.open("w") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print("repair: an ingest is running; try again when it finishes.", + file=sys.stderr) + return 2 + + try: + stale: list[tuple[Path, str, str]] = [] + damaged: list[Path] = [] + checked = 0 + for sess_dir, manifest in iter_all_sessions(archive): + zst = sess_dir / COMPRESSED_TRANSCRIPT + recorded = manifest.get("sha256_compressed") + if not zst.exists() or not recorded: + continue + checked += 1 + raw = zst.read_bytes() + actual = sha256_bytes(raw) + if actual == recorded: + continue + try: + canonical = sha256_bytes(decompress_bytes(raw)) + except Exception: + canonical = None + if canonical and canonical == manifest.get("sha256"): + stale.append((sess_dir, recorded, actual)) + else: + damaged.append(sess_dir) + + rel = lambda d: d.relative_to(archive) # noqa: E731 + print(f"repair: checked {checked} compressed deposit(s): " + f"{len(stale)} stale hash(es), {len(damaged)} damaged") + for d in damaged: + print(f" DAMAGED (not rewritten): {rel(d)}", file=sys.stderr) + + if args.dry_run or not stale: + return 1 if damaged else 0 + + for sess_dir, old, new in stale: + path = sess_dir / "manifest.json" + text = path.read_text() + if text.count(old) != 1: + print(f" skipped (hash not found exactly once): {rel(sess_dir)}", + file=sys.stderr) + continue + tmp = path.with_suffix(".json.partial") + tmp.write_text(text.replace(old, new)) + os.replace(tmp, path) + + paths = [f"{rel(d)}/manifest.json" for d, _, _ in stale] + for i in range(0, len(paths), 500): + run_git(archive, "add", *paths[i:i + 500]) + if run_git(archive, "diff", "--cached", "--quiet").returncode != 0: + config_path = archive / ".holotype" / "config.json" + try: + config = json.loads(config_path.read_text()) + except (OSError, json.JSONDecodeError): + config = {} + sign = bool((config.get("deposit") or {}).get("sign_commits")) + msg = ( + f"repair: rewrite {len(stale)} stale sha256_compressed value(s)\n\n" + "Each manifest's sha256_compressed no longer matched its .zst\n" + "file. The decompressed content still matches the canonical\n" + "sha256, so only the compressed hash was rewritten. Transcripts\n" + "and the hash-chain ledger are unchanged." + ) + commit_args = ["commit", "-m", msg] + if sign: + commit_args.insert(1, "-S") + out = run_git(archive, *commit_args) + if out.returncode != 0: + print(f"repair: commit failed: {out.stderr.strip()}", file=sys.stderr) + return 2 + print(f"repair: committed {len(stale)} manifest(s)") + return 1 if damaged else 0 + finally: + fcntl.flock(lock, fcntl.LOCK_UN) + lock.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/selftest.py b/scripts/selftest.py index c4f586d..a467826 100644 --- a/scripts/selftest.py +++ b/scripts/selftest.py @@ -171,6 +171,20 @@ def main(argv: list[str] | None = None) -> int: expect((archive / "README.md").exists(), "archive README.md missing") expect((archive / ".git").is_dir(), "archive is not a git repo") + step("init.py turns off git auto-maintenance in the archive") + for key, want in (("maintenance.auto", "false"), ("gc.auto", "0")): + got = subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--get", key], + capture_output=True, text=True, + ).stdout.strip() + expect(got == want, f"archive {key}={got!r}, expected {want!r}") + # Unset again so the first ingest below has to migrate an + # archive created before this setting existed. + subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--unset", key], + check=True, + ) + step("materialize synthetic source") source = materialize_source(tmp) @@ -186,6 +200,14 @@ def main(argv: list[str] | None = None) -> int: expect(result.returncode == 0, f"ingest.py returncode={result.returncode}: {result.stderr}") expect("new=3" in result.stdout, f"expected new=3 in output:\n{result.stdout}") + step("ingest turns off git auto-maintenance on an older archive") + for key, want in (("maintenance.auto", "false"), ("gc.auto", "0")): + got = subprocess.run( + ["git", "-C", str(archive), "config", "--local", "--get", key], + capture_output=True, text=True, + ).stdout.strip() + expect(got == want, f"archive {key}={got!r} after ingest, expected {want!r}") + sessions_dir = archive / "sessions" deposits = sorted(p for p in sessions_dir.rglob("transcript.jsonl")) expect(len(deposits) == 3, f"expected 3 deposits (2 top-level + 1 subagent), got {len(deposits)}") @@ -1559,6 +1581,55 @@ def main(argv: list[str] | None = None) -> int: expect(res.returncode == 0, f"verify failed after pure migration: {res.stderr or res.stdout}") + step("stale sha256_compressed: migration rehashes, fsck and repair tell it from damage") + # A stale recorded compressed hash (content intact) must not + # survive a migration: the migration hashes the file on disk. + zc_obj = json.loads(zc_manifest.read_text()) + zc_obj["manifest_version"] = 3 + zc_obj["sha256_compressed"] = "0" * 64 + zc_manifest.write_text(json.dumps(zc_obj, indent=2, sort_keys=True) + "\n") + res = run_script( + REPO_ROOT / "scripts" / "ingest.py", "--archive", str(zc_archive), + "--source", str(zc_src), "--source-name", "claude-code", "--workers", "1", + ) + expect(res.returncode == 0, f"zc stale-hash migration failed: {res.stderr}") + post = json.loads(zc_manifest.read_text()) + expect(post.get("sha256_compressed") == hashlib.sha256(zc_zst.read_bytes()).hexdigest(), + "migration carried a stale sha256_compressed forward") + + # The fsck reports a stale hash as stale, not as an integrity failure. + zc_obj = json.loads(zc_manifest.read_text()) + real_hash = zc_obj["sha256_compressed"] + zc_obj["sha256_compressed"] = "0" * 64 + zc_manifest.write_text(json.dumps(zc_obj, indent=2, sort_keys=True) + "\n") + # Commit the stale value, as it would be in a real archive. + subprocess.run(["git", "-C", str(zc_archive), "commit", "-qam", "test: stale hash"], + check=True) + stale: list[str] = [] + fsck_err = _ig.verify_deposit_integrity(zc_archive, zc_cand, stale) + expect(fsck_err is None and stale == [zc_cand.archive_subpath], + f"stale compressed hash misreported (err={fsck_err!r}, stale={stale})") + + # The repair script rewrites it and commits; verify is clean after. + res = run_script(REPO_ROOT / "scripts" / "repair_compressed_hashes.py", + "--archive", str(zc_archive)) + expect(res.returncode == 0, f"repair failed: {res.stderr or res.stdout}") + expect(json.loads(zc_manifest.read_text())["sha256_compressed"] == real_hash, + "repair did not restore the real sha256_compressed") + log = subprocess.run(["git", "-C", str(zc_archive), "log", "-1", "--format=%s"], + capture_output=True, text=True).stdout + expect(log.startswith("repair:"), f"repair did not commit (HEAD: {log!r})") + + # Real damage is still an integrity failure and is never rewritten. + zc_zst.write_bytes(b"not zstd at all") + stale = [] + fsck_err = _ig.verify_deposit_integrity(zc_archive, zc_cand, stale) + expect(fsck_err is not None and "INTEGRITY" in fsck_err and not stale, + f"damaged deposit not reported as failure (err={fsck_err!r})") + res = run_script(REPO_ROOT / "scripts" / "repair_compressed_hashes.py", + "--archive", str(zc_archive)) + expect(res.returncode == 1, f"repair should exit 1 on damage, got {res.returncode}") + print("\nALL CHECKS PASSED") success = True return 0