feat: add --audit and --summary to diff_results.py

README.md

@@ -1227,7 +1227,7 @@ pip install -r requirements-lock.txt
 
 **For new audits and reruns:** Update `results/` as usual. Do not regenerate `paper/results-frozen/` or move the `v1.0-paper` tag as part of an ordinary audit. Run `scripts/freeze_paper_results.py` only when deliberately creating a new reference snapshot; it replaces the existing frozen snapshot and mirrors its CSV files into `faircode/_results_frozen/`.
 
-To see how far `results/` has moved from that snapshot, run `python3 scripts/diff_results.py`. It joins the two tables per audit, strategy, model, protected attribute and metric and prints the per-row change, flagging rows that moved by more than `--threshold` (default 0.01) or exist on one side only (`--json`, `--all`, `--kind` and `--fail-on-change` are also available; it only reads).
+To see how far `results/` has moved from that snapshot, run `python3 scripts/diff_results.py`. It joins the two tables per audit, strategy, model, protected attribute and metric and prints the per-row change, flagging rows that moved by more than `--threshold` (default 0.01) or exist on one side only (`--json`, `--all`, `--kind` and `--fail-on-change` are also available; `--audit NAME` limits it to one or more audits and `--summary` prints one line per audit; it only reads).
 
 To deliberately create a new reference snapshot, run:
 

faircode/SPEC.md

@@ -604,7 +604,9 @@ issue #388, verified by building an actual wheel and installing it in a clean ve
 reports each row as `moved` (|current − reference| greater than `--threshold`, default 0.01, or a
 value empty on one side only), `unchanged`, `only_current` or `only_reference`. Rows present on one
 side only are always flagged. `--json` prints the structured diff, `--all` lists unchanged rows, and
-`--fail-on-change` exits 1 when anything is flagged. It only reads both directories.
+`--fail-on-change` exits 1 when anything is flagged. `--audit NAME` (repeatable) restricts the
+join to those audits, and `--summary` prints one line per audit and kind instead (rows compared,
+rows flagged, largest absolute delta); both work with `--json`. It only reads both directories.
 
 ---
 

scripts/diff_results.py

@@ -10,6 +10,8 @@
     python3 scripts/diff_results.py --kind fairness --all # every row, not just the ones that moved
     python3 scripts/diff_results.py --json                # machine-readable
     python3 scripts/diff_results.py --fail-on-change      # exit 1 if any row is flagged (for CI)
+    python3 scripts/diff_results.py --audit COMPAS        # only these audits (repeatable)
+    python3 scripts/diff_results.py --summary             # one line per audit and kind
 
 Join keys: fairness = audit, strategy, model, protected_attribute, metric; performance = audit,
 strategy, model, metric. A row present on one side only is reported as `only_current` or
@@ -72,18 +74,63 @@ def index(df):
 
 
 def diff_dirs(current_dir: Path, reference_dir: Path, kinds=("fairness", "performance"),
-              threshold: float = DEFAULT_THRESHOLD) -> dict:
+              threshold: float = DEFAULT_THRESHOLD, audits=None) -> dict:
+    """With `audits`, only rows whose audit is one of those names are joined."""
     out = {}
     for kind in kinds:
         filename, keys = KINDS[kind]
         cur_path, ref_path = Path(current_dir) / filename, Path(reference_dir) / filename
         for p in (cur_path, ref_path):
             if not p.is_file():
                 raise FileNotFoundError(f"{p} not found")
-        out[kind] = diff_table(pd.read_csv(cur_path), pd.read_csv(ref_path), keys, threshold)
+        current, reference = pd.read_csv(cur_path), pd.read_csv(ref_path)
+        if audits:
+            wanted = set(audits)
+            current = current[current["audit"].astype(str).isin(wanted)]
+            reference = reference[reference["audit"].astype(str).isin(wanted)]
+        out[kind] = diff_table(current, reference, keys, threshold)
     return out
 
 
+def known_audits(current_dir: Path, reference_dir: Path, kinds=("fairness", "performance")) -> set[str]:
+    """Every audit name on either side, for checking --audit."""
+    names = set()
+    for kind in kinds:
+        filename = KINDS[kind][0]
+        for directory in (current_dir, reference_dir):
+            path = Path(directory) / filename
+            if path.is_file():
+                names.update(pd.read_csv(path)["audit"].astype(str))
+    return names
+
+
+def summarize(diff: dict) -> dict:
+    """Per kind, one dict per audit: rows compared, rows flagged, largest absolute delta (None if no numeric delta)."""
+    out = {}
+    for kind, rows in diff.items():
+        per_audit = {}
+        for r in rows:
+            s = per_audit.setdefault(r["audit"], {"audit": r["audit"], "compared": 0, "flagged": 0,
+                                                  "max_abs_delta": None})
+            s["compared"] += 1
+            s["flagged"] += _flagged(r)
+            if r["delta"] is not None and (s["max_abs_delta"] is None or abs(r["delta"]) > s["max_abs_delta"]):
+                s["max_abs_delta"] = abs(r["delta"])
+        out[kind] = [per_audit[a] for a in sorted(per_audit)]
+    return out
+
+
+def render_summary(summary: dict) -> str:
+    lines = []
+    for kind, audits in summary.items():
+        lines.append(f"{kind}:")
+        for s in audits:
+            largest = "n/a" if s["max_abs_delta"] is None else f"{s['max_abs_delta']:.4f}"
+            lines.append(f"  {s['audit']}: {s['compared']} compared, {s['flagged']} flagged, largest |delta| {largest}")
+        lines.append("")
+    return "\n".join(lines).rstrip() + "\n"
+
+
 def _flagged(row: dict) -> bool:
     return row["status"] != "unchanged"
 
@@ -124,16 +171,28 @@ def main(argv=None) -> int:
     p.add_argument("--all", action="store_true", help="list unchanged rows too")
     p.add_argument("--json", action="store_true", help="print the diff as JSON")
     p.add_argument("--fail-on-change", action="store_true", help="exit 1 if any row is flagged")
+    p.add_argument("--audit", action="append", metavar="NAME",
+                   help="only compare this audit (repeatable; default: every audit)")
+    p.add_argument("--summary", action="store_true",
+                   help="print one line per audit and kind instead of every row")
     args = p.parse_args(argv)
     if args.threshold < 0:
         p.error("--threshold must be zero or positive")
     kinds = ("fairness", "performance") if args.kind == "both" else (args.kind,)
     try:
-        diff = diff_dirs(args.current, args.reference, kinds, args.threshold)
+        if args.audit:
+            unknown = sorted(set(args.audit) - known_audits(args.current, args.reference, kinds))
+            if unknown:
+                print(f"error: unknown audit(s): {', '.join(unknown)}", file=sys.stderr)
+                return 2
+        diff = diff_dirs(args.current, args.reference, kinds, args.threshold, args.audit)
     except FileNotFoundError as exc:
         print(f"error: {exc}", file=sys.stderr)
         return 2
-    if args.json:
+    if args.summary:
+        summary = summarize(diff)
+        print(json.dumps(summary, indent=2) if args.json else render_summary(summary), end="\n" if args.json else "")
+    elif args.json:
         print(json.dumps(diff, indent=2))
     else:
         print(render(diff, args.all, args.threshold), end="")

tests/test_diff_results.py

@@ -84,3 +84,47 @@ def test_the_real_snapshot_and_current_results_compare():
     diff = dr.diff_dirs(dr.CURRENT_DIR, dr.REFERENCE_DIR)
     assert set(diff) == {"fairness", "performance"}
     assert diff["fairness"] and diff["performance"]
+
+
+def _two_audits(tmp_path):
+    ref = _fair([["a", "baseline", "lr", "sex", "dp", 0.10], ["b", "baseline", "lr", "sex", "dp", 0.10],
+                 ["b", "baseline", "lr", "sex", "eo", 0.20]])
+    cur = _fair([["a", "baseline", "lr", "sex", "dp", 0.10], ["b", "baseline", "lr", "sex", "dp", 0.30],
+                 ["b", "baseline", "lr", "sex", "eo", 0.15]])
+    _write(tmp_path / "ref", ref)
+    _write(tmp_path / "cur", cur)
+    return ["--current", str(tmp_path / "cur"), "--reference", str(tmp_path / "ref"), "--kind", "fairness"]
+
+
+def test_audit_restricts_the_join_and_is_repeatable(tmp_path, capsys):
+    args = _two_audits(tmp_path)
+    assert dr.main([*args, "--audit", "a", "--json", "--all"]) == 0
+    rows = json.loads(capsys.readouterr().out)["fairness"]
+    assert {r["audit"] for r in rows} == {"a"}
+    assert dr.main([*args, "--audit", "a", "--audit", "b", "--json", "--all"]) == 0
+    assert len(json.loads(capsys.readouterr().out)["fairness"]) == 3
+
+
+def test_unknown_audit_is_an_error(tmp_path, capsys):
+    assert dr.main([*_two_audits(tmp_path), "--audit", "nope"]) == 2
+    assert "unknown audit(s): nope" in capsys.readouterr().err
+
+
+def test_summary_counts_rows_and_largest_delta_per_audit(tmp_path, capsys):
+    args = _two_audits(tmp_path)
+    assert dr.main([*args, "--summary", "--json"]) == 0
+    summary = json.loads(capsys.readouterr().out)["fairness"]
+    assert [s["audit"] for s in summary] == ["a", "b"]
+    assert summary[0] == {"audit": "a", "compared": 1, "flagged": 0, "max_abs_delta": 0.0}
+    assert (summary[1]["compared"], summary[1]["flagged"]) == (2, 2)
+    assert summary[1]["max_abs_delta"] == pytest.approx(0.2)
+
+    assert dr.main([*args, "--summary", "--audit", "b"]) == 0
+    out = capsys.readouterr().out
+    assert "b: 2 compared, 2 flagged, largest |delta| 0.2000" in out
+    assert "a:" not in out
+
+
+def test_summary_largest_delta_is_none_without_numeric_deltas():
+    diff = {"fairness": [{"audit": "a", "status": "only_current", "delta": None}]}
+    assert dr.summarize(diff) == {"fairness": [{"audit": "a", "compared": 1, "flagged": 1, "max_abs_delta": None}]}