diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0cf164e..a1d13d9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -459,7 +459,9 @@ A short sample, not a full list. README.md and llms.txt list every explainer, an The `assets/explainers-data.json` entry has `slug`, `title`, `subtitle`, `summary` and `tags`, and two optional fields the build checks: `description` (the meta description, 70 to 160 characters; when it is missing the summary is cut at the last sentence that fits, and a summary that cannot be shortened sensibly fails the build) and `seo_title` (a shorter page title when `title` plus the site name would pass 65 characters, so at most 53). -Explainers with code should run against a real audit. `faircode.benchmark.load_audit("Benefits Denial")` loads an audit with the same loader, encoding and train/test split as the benchmark harness, so a baseline in your explainer matches `results/`. +Explainers with code should run against a real audit. To opt a Python fence into the smoke test, put `# runnable` on the first nonblank line inside the fence. The test executes each marked block in a subprocess from the repository root, reports the explainer and block number on failure, and applies a timeout. Add `# slow` on the next line for examples that need the longer timeout and should be skipped by `make test-fast`; the full CI test suite still runs them. Keep setup self-contained and avoid network access. + + `faircode.benchmark.load_audit("Benefits Denial")` loads an audit with the same loader, encoding and train/test split as the benchmark harness, so a baseline in your explainer matches `results/`. File naming: diff --git a/explainers/predictive-parity.md b/explainers/predictive-parity.md index b830c9f..3369953 100644 --- a/explainers/predictive-parity.md +++ b/explainers/predictive-parity.md @@ -35,6 +35,7 @@ Chouldechova (2017) proved the mechanism algebraically: if predictive parity hol Compute Positive Predictive Value per group, then pair it with a base-rate check, since a small PPV gap next to a large base-rate gap is the signature Chouldechova identified. ```python +# runnable import pandas as pd @@ -84,9 +85,15 @@ def base_rate_gap(df: pd.DataFrame, y_true: str, group_col: str): return rates.max() - rates.min() -# Usage example -# predictive_parity_gap(compas_df, y_true="two_year_recid", y_pred="flagged_high_risk", group_col="race") -# base_rate_gap(compas_df, y_true="two_year_recid", group_col="race") +sample = pd.DataFrame( + { + "group": ["A", "A", "B", "B"], + "y_true": [1, 0, 1, 0], + "y_pred": [1, 1, 1, 0], + } +) +predictive_parity_gap(sample, y_true="y_true", y_pred="y_pred", group_col="group") +base_rate_gap(sample, y_true="y_true", group_col="group") ``` ## Concrete Example: COMPAS - Audit 01 diff --git a/tests/test_explainer_code_smoke.py b/tests/test_explainer_code_smoke.py new file mode 100644 index 0000000..1784579 --- /dev/null +++ b/tests/test_explainer_code_smoke.py @@ -0,0 +1,66 @@ +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +EXPLAINERS_DIR = REPO_ROOT / "explainers" +PYTHON_BLOCK = re.compile(r"^```python[^\n]*\n(.*?)^```\s*$", re.MULTILINE | re.DOTALL) +RUNNABLE_MARKER = "# runnable" +SLOW_MARKER = "# slow" + + +def runnable_blocks(): + for path in sorted(EXPLAINERS_DIR.glob("*.md")): + content = path.read_text(encoding="utf-8") + for block_number, match in enumerate(PYTHON_BLOCK.finditer(content), start=1): + lines = match.group(1).splitlines() + first_content = next((line.strip() for line in lines if line.strip()), "") + if first_content != RUNNABLE_MARKER: + continue + slow = SLOW_MARKER in [line.strip() for line in lines[:3]] + code = "\n".join( + line for line in lines if line.strip() not in {RUNNABLE_MARKER, SLOW_MARKER} + ) + marks = [pytest.mark.slow] if slow else [] + yield pytest.param( + path, + block_number, + code, + slow, + id=f"{path.name}-python-block-{block_number}", + marks=marks, + ) + + +@pytest.mark.parametrize("path,block_number,code,slow", list(runnable_blocks())) +def test_runnable_explainer_python_blocks(path, block_number, code, slow): + timeout = 120 if slow else 30 + try: + result = subprocess.run( + [sys.executable, "-c", code], + cwd=REPO_ROOT, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + pytest.fail( + f"{path.relative_to(REPO_ROOT)} Python block {block_number} timed out " + f"after {timeout}s.\nstdout: {exc.stdout or ''}\nstderr: {exc.stderr or ''}" + ) + + assert result.returncode == 0, ( + f"{path.relative_to(REPO_ROOT)} Python block {block_number} exited " + f"with status {result.returncode}.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +def test_at_least_one_explainer_python_block_is_marked_runnable(): + assert list(runnable_blocks()), ( + "No runnable explainer Python blocks found. Add '# runnable' as the first " + "nonblank line inside a Python fence in explainers/*.md." + )