diff --git a/README.md b/README.md index d82631d..6620158 100644 --- a/README.md +++ b/README.md @@ -1227,7 +1227,7 @@ pip install -r requirements-lock.txt **For new audits and reruns:** Update `results/` as usual. Do not regenerate `paper/results-frozen/` or move the `v1.0-paper` tag as part of an ordinary audit. Run `scripts/freeze_paper_results.py` only when deliberately creating a new reference snapshot; it replaces the existing frozen snapshot and mirrors its CSV files into `faircode/_results_frozen/`. -To see how far `results/` has moved from that snapshot, run `python3 scripts/diff_results.py`. It joins the two tables per audit, strategy, model, protected attribute and metric and prints the per-row change, flagging rows that moved by more than `--threshold` (default 0.01) or exist on one side only (`--json`, `--all`, `--kind` and `--fail-on-change` are also available; it only reads). +To see how far `results/` has moved from that snapshot, run `python3 scripts/diff_results.py`. It joins the two tables per audit, strategy, model, protected attribute and metric and prints the per-row change, flagging rows that moved by more than `--threshold` (default 0.01) or exist on one side only (`--json`, `--all`, `--kind` and `--fail-on-change` are also available; `--audit NAME` limits it to one or more audits and `--summary` prints one line per audit; it only reads). To deliberately create a new reference snapshot, run: diff --git a/faircode/SPEC.md b/faircode/SPEC.md index 4e48994..819c243 100644 --- a/faircode/SPEC.md +++ b/faircode/SPEC.md @@ -604,7 +604,9 @@ issue #388, verified by building an actual wheel and installing it in a clean ve reports each row as `moved` (|current − reference| greater than `--threshold`, default 0.01, or a value empty on one side only), `unchanged`, `only_current` or `only_reference`. Rows present on one side only are always flagged. `--json` prints the structured diff, `--all` lists unchanged rows, and -`--fail-on-change` exits 1 when anything is flagged. It only reads both directories. +`--fail-on-change` exits 1 when anything is flagged. `--audit NAME` (repeatable) restricts the +join to those audits, and `--summary` prints one line per audit and kind instead (rows compared, +rows flagged, largest absolute delta); both work with `--json`. It only reads both directories. --- diff --git a/scripts/diff_results.py b/scripts/diff_results.py index 0536aa7..18e8f97 100644 --- a/scripts/diff_results.py +++ b/scripts/diff_results.py @@ -10,6 +10,8 @@ python3 scripts/diff_results.py --kind fairness --all # every row, not just the ones that moved python3 scripts/diff_results.py --json # machine-readable python3 scripts/diff_results.py --fail-on-change # exit 1 if any row is flagged (for CI) + python3 scripts/diff_results.py --audit COMPAS # only these audits (repeatable) + python3 scripts/diff_results.py --summary # one line per audit and kind Join keys: fairness = audit, strategy, model, protected_attribute, metric; performance = audit, strategy, model, metric. A row present on one side only is reported as `only_current` or @@ -72,7 +74,8 @@ def index(df): def diff_dirs(current_dir: Path, reference_dir: Path, kinds=("fairness", "performance"), - threshold: float = DEFAULT_THRESHOLD) -> dict: + threshold: float = DEFAULT_THRESHOLD, audits=None) -> dict: + """With `audits`, only rows whose audit is one of those names are joined.""" out = {} for kind in kinds: filename, keys = KINDS[kind] @@ -80,10 +83,54 @@ def diff_dirs(current_dir: Path, reference_dir: Path, kinds=("fairness", "perfor for p in (cur_path, ref_path): if not p.is_file(): raise FileNotFoundError(f"{p} not found") - out[kind] = diff_table(pd.read_csv(cur_path), pd.read_csv(ref_path), keys, threshold) + current, reference = pd.read_csv(cur_path), pd.read_csv(ref_path) + if audits: + wanted = set(audits) + current = current[current["audit"].astype(str).isin(wanted)] + reference = reference[reference["audit"].astype(str).isin(wanted)] + out[kind] = diff_table(current, reference, keys, threshold) return out +def known_audits(current_dir: Path, reference_dir: Path, kinds=("fairness", "performance")) -> set[str]: + """Every audit name on either side, for checking --audit.""" + names = set() + for kind in kinds: + filename = KINDS[kind][0] + for directory in (current_dir, reference_dir): + path = Path(directory) / filename + if path.is_file(): + names.update(pd.read_csv(path)["audit"].astype(str)) + return names + + +def summarize(diff: dict) -> dict: + """Per kind, one dict per audit: rows compared, rows flagged, largest absolute delta (None if no numeric delta).""" + out = {} + for kind, rows in diff.items(): + per_audit = {} + for r in rows: + s = per_audit.setdefault(r["audit"], {"audit": r["audit"], "compared": 0, "flagged": 0, + "max_abs_delta": None}) + s["compared"] += 1 + s["flagged"] += _flagged(r) + if r["delta"] is not None and (s["max_abs_delta"] is None or abs(r["delta"]) > s["max_abs_delta"]): + s["max_abs_delta"] = abs(r["delta"]) + out[kind] = [per_audit[a] for a in sorted(per_audit)] + return out + + +def render_summary(summary: dict) -> str: + lines = [] + for kind, audits in summary.items(): + lines.append(f"{kind}:") + for s in audits: + largest = "n/a" if s["max_abs_delta"] is None else f"{s['max_abs_delta']:.4f}" + lines.append(f" {s['audit']}: {s['compared']} compared, {s['flagged']} flagged, largest |delta| {largest}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + def _flagged(row: dict) -> bool: return row["status"] != "unchanged" @@ -124,16 +171,28 @@ def main(argv=None) -> int: p.add_argument("--all", action="store_true", help="list unchanged rows too") p.add_argument("--json", action="store_true", help="print the diff as JSON") p.add_argument("--fail-on-change", action="store_true", help="exit 1 if any row is flagged") + p.add_argument("--audit", action="append", metavar="NAME", + help="only compare this audit (repeatable; default: every audit)") + p.add_argument("--summary", action="store_true", + help="print one line per audit and kind instead of every row") args = p.parse_args(argv) if args.threshold < 0: p.error("--threshold must be zero or positive") kinds = ("fairness", "performance") if args.kind == "both" else (args.kind,) try: - diff = diff_dirs(args.current, args.reference, kinds, args.threshold) + if args.audit: + unknown = sorted(set(args.audit) - known_audits(args.current, args.reference, kinds)) + if unknown: + print(f"error: unknown audit(s): {', '.join(unknown)}", file=sys.stderr) + return 2 + diff = diff_dirs(args.current, args.reference, kinds, args.threshold, args.audit) except FileNotFoundError as exc: print(f"error: {exc}", file=sys.stderr) return 2 - if args.json: + if args.summary: + summary = summarize(diff) + print(json.dumps(summary, indent=2) if args.json else render_summary(summary), end="\n" if args.json else "") + elif args.json: print(json.dumps(diff, indent=2)) else: print(render(diff, args.all, args.threshold), end="") diff --git a/tests/test_diff_results.py b/tests/test_diff_results.py index 9737eb8..2faf9f3 100644 --- a/tests/test_diff_results.py +++ b/tests/test_diff_results.py @@ -84,3 +84,47 @@ def test_the_real_snapshot_and_current_results_compare(): diff = dr.diff_dirs(dr.CURRENT_DIR, dr.REFERENCE_DIR) assert set(diff) == {"fairness", "performance"} assert diff["fairness"] and diff["performance"] + + +def _two_audits(tmp_path): + ref = _fair([["a", "baseline", "lr", "sex", "dp", 0.10], ["b", "baseline", "lr", "sex", "dp", 0.10], + ["b", "baseline", "lr", "sex", "eo", 0.20]]) + cur = _fair([["a", "baseline", "lr", "sex", "dp", 0.10], ["b", "baseline", "lr", "sex", "dp", 0.30], + ["b", "baseline", "lr", "sex", "eo", 0.15]]) + _write(tmp_path / "ref", ref) + _write(tmp_path / "cur", cur) + return ["--current", str(tmp_path / "cur"), "--reference", str(tmp_path / "ref"), "--kind", "fairness"] + + +def test_audit_restricts_the_join_and_is_repeatable(tmp_path, capsys): + args = _two_audits(tmp_path) + assert dr.main([*args, "--audit", "a", "--json", "--all"]) == 0 + rows = json.loads(capsys.readouterr().out)["fairness"] + assert {r["audit"] for r in rows} == {"a"} + assert dr.main([*args, "--audit", "a", "--audit", "b", "--json", "--all"]) == 0 + assert len(json.loads(capsys.readouterr().out)["fairness"]) == 3 + + +def test_unknown_audit_is_an_error(tmp_path, capsys): + assert dr.main([*_two_audits(tmp_path), "--audit", "nope"]) == 2 + assert "unknown audit(s): nope" in capsys.readouterr().err + + +def test_summary_counts_rows_and_largest_delta_per_audit(tmp_path, capsys): + args = _two_audits(tmp_path) + assert dr.main([*args, "--summary", "--json"]) == 0 + summary = json.loads(capsys.readouterr().out)["fairness"] + assert [s["audit"] for s in summary] == ["a", "b"] + assert summary[0] == {"audit": "a", "compared": 1, "flagged": 0, "max_abs_delta": 0.0} + assert (summary[1]["compared"], summary[1]["flagged"]) == (2, 2) + assert summary[1]["max_abs_delta"] == pytest.approx(0.2) + + assert dr.main([*args, "--summary", "--audit", "b"]) == 0 + out = capsys.readouterr().out + assert "b: 2 compared, 2 flagged, largest |delta| 0.2000" in out + assert "a:" not in out + + +def test_summary_largest_delta_is_none_without_numeric_deltas(): + diff = {"fairness": [{"audit": "a", "status": "only_current", "delta": None}]} + assert dr.summarize(diff) == {"fairness": [{"audit": "a", "compared": 1, "flagged": 1, "max_abs_delta": None}]}