From 5802e3705dcdc62d506924d9c25a281bf3612798 Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Sun, 5 Jul 2026 13:18:10 -0400 Subject: [PATCH 1/5] Add sample fixture drift checker --- scripts/check_sample_fixtures.py | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 scripts/check_sample_fixtures.py diff --git a/scripts/check_sample_fixtures.py b/scripts/check_sample_fixtures.py new file mode 100644 index 00000000..257d56ac --- /dev/null +++ b/scripts/check_sample_fixtures.py @@ -0,0 +1,107 @@ +"""Check or refresh generated dataset sample fixtures. + +The check regenerates ``sample_std.json`` from each dataset's checked-in +``sample_atif.json`` and compares it with the committed fixture. +""" + +from __future__ import annotations + +import argparse +import difflib +import json +import os +import subprocess +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DATASETS_DIR = REPO_ROOT / "datasets" + + +def dataset_dirs(names: list[str]) -> list[Path]: + if names: + return [DATASETS_DIR / name for name in names] + return sorted(path for path in DATASETS_DIR.iterdir() if path.is_dir()) + + +def render_jsonl(items: list[object]) -> str: + return "".join(json.dumps(item, ensure_ascii=False) + "\n" for item in items) + + +def regenerate_sample_std(dataset_dir: Path) -> str: + sample_atif = dataset_dir / "sample_atif.json" + converter = dataset_dir / "atif_to_std.py" + if not sample_atif.exists() or not converter.exists(): + raise FileNotFoundError(f"{dataset_dir.name} is missing sample_atif.json or atif_to_std.py") + + env = os.environ.copy() + env["PYTHONPATH"] = str(REPO_ROOT) + atif_rows = json.loads(sample_atif.read_text()) + result = subprocess.run( + [sys.executable, str(converter)], + input=render_jsonl(atif_rows), + text=True, + capture_output=True, + check=True, + env=env, + ) + std_rows = [json.loads(line) for line in result.stdout.splitlines() if line.strip()] + return json.dumps(std_rows, indent=2, ensure_ascii=True) + "\n" + + +def check_dataset(dataset_dir: Path, update: bool) -> tuple[bool, str]: + sample_std = dataset_dir / "sample_std.json" + if not sample_std.exists(): + return True, f"skip {dataset_dir.name}: missing sample_std.json" + + regenerated = regenerate_sample_std(dataset_dir) + current = sample_std.read_text() + if regenerated == current: + return True, f"ok {dataset_dir.name}" + + if update: + sample_std.write_text(regenerated) + return True, f"updated {dataset_dir.name}" + + diff = "".join( + difflib.unified_diff( + current.splitlines(keepends=True), + regenerated.splitlines(keepends=True), + fromfile=f"{dataset_dir.name}/sample_std.json", + tofile=f"{dataset_dir.name}/sample_std.json regenerated", + n=3, + ) + ) + return False, f"drift {dataset_dir.name}\n{diff}" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dataset", + action="append", + default=[], + help="Dataset directory name to check. May be repeated. Defaults to all datasets.", + ) + parser.add_argument( + "--update", + action="store_true", + help="Rewrite stale sample_std.json files instead of failing.", + ) + args = parser.parse_args() + + ok = True + for dataset_dir in dataset_dirs(args.dataset): + try: + dataset_ok, message = check_dataset(dataset_dir, args.update) + except Exception as exc: + dataset_ok = False + message = f"error {dataset_dir.name}: {exc}" + ok = ok and dataset_ok + print(message) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From ca908db853149a0a969e1d9cf00d01e06425f8ca Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Sun, 5 Jul 2026 14:22:08 -0400 Subject: [PATCH 2/5] Apply pre-commit fixes --- scripts/check_sample_fixtures.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/check_sample_fixtures.py b/scripts/check_sample_fixtures.py index 257d56ac..7fb68eab 100644 --- a/scripts/check_sample_fixtures.py +++ b/scripts/check_sample_fixtures.py @@ -14,7 +14,6 @@ import sys from pathlib import Path - REPO_ROOT = Path(__file__).resolve().parents[1] DATASETS_DIR = REPO_ROOT / "datasets" From 199e48ac46599279b11f2908da298072c95fbc5a Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Sun, 5 Jul 2026 14:56:22 -0400 Subject: [PATCH 3/5] Harden sample fixture checker errors --- scripts/check_sample_fixtures.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/check_sample_fixtures.py b/scripts/check_sample_fixtures.py index 7fb68eab..90340cae 100644 --- a/scripts/check_sample_fixtures.py +++ b/scripts/check_sample_fixtures.py @@ -20,7 +20,11 @@ def dataset_dirs(names: list[str]) -> list[Path]: if names: - return [DATASETS_DIR / name for name in names] + dirs = [DATASETS_DIR / name for name in names] + for dataset_dir in dirs: + if not dataset_dir.is_dir(): + raise SystemExit(f"error: unknown dataset '{dataset_dir.name}'") + return dirs return sorted(path for path in DATASETS_DIR.iterdir() if path.is_dir()) @@ -35,7 +39,10 @@ def regenerate_sample_std(dataset_dir: Path) -> str: raise FileNotFoundError(f"{dataset_dir.name} is missing sample_atif.json or atif_to_std.py") env = os.environ.copy() - env["PYTHONPATH"] = str(REPO_ROOT) + existing_pythonpath = os.environ.get("PYTHONPATH", "") + env["PYTHONPATH"] = os.pathsep.join( + entry for entry in [str(REPO_ROOT), existing_pythonpath] if entry + ) atif_rows = json.loads(sample_atif.read_text()) result = subprocess.run( [sys.executable, str(converter)], @@ -94,6 +101,9 @@ def main() -> int: for dataset_dir in dataset_dirs(args.dataset): try: dataset_ok, message = check_dataset(dataset_dir, args.update) + except subprocess.CalledProcessError as exc: + dataset_ok = False + message = f"error {dataset_dir.name}: converter failed\n{exc.stderr}" except Exception as exc: dataset_ok = False message = f"error {dataset_dir.name}: {exc}" From 5b1ba5d9607c0aab5b6f64ce2bafae7b93c2dd48 Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Sun, 5 Jul 2026 16:38:52 -0400 Subject: [PATCH 4/5] Match sample fixture JSONL serialization --- scripts/check_sample_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_sample_fixtures.py b/scripts/check_sample_fixtures.py index 90340cae..fd5c76c5 100644 --- a/scripts/check_sample_fixtures.py +++ b/scripts/check_sample_fixtures.py @@ -29,7 +29,7 @@ def dataset_dirs(names: list[str]) -> list[Path]: def render_jsonl(items: list[object]) -> str: - return "".join(json.dumps(item, ensure_ascii=False) + "\n" for item in items) + return "".join(json.dumps(item, ensure_ascii=True) + "\n" for item in items) def regenerate_sample_std(dataset_dir: Path) -> str: From ab3785cdc78aa0459ca8deb1fdcaaa8e7a0614d9 Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Sun, 5 Jul 2026 16:45:25 -0400 Subject: [PATCH 5/5] Warn when fixture update may stale SFT samples --- scripts/check_sample_fixtures.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/check_sample_fixtures.py b/scripts/check_sample_fixtures.py index fd5c76c5..63603e4b 100644 --- a/scripts/check_sample_fixtures.py +++ b/scripts/check_sample_fixtures.py @@ -68,7 +68,13 @@ def check_dataset(dataset_dir: Path, update: bool) -> tuple[bool, str]: if update: sample_std.write_text(regenerated) - return True, f"updated {dataset_dir.name}" + sft_dir = dataset_dir / "sample_sft" + sft_warning = ( + " (re-run agents/*/std_to_sft.py to refresh sample_sft/)" + if sft_dir.is_dir() + else "" + ) + return True, f"updated {dataset_dir.name}{sft_warning}" diff = "".join( difflib.unified_diff(