Run executable controls for the verdict evidence predicate

.github/workflows/verdict-evidence-controls.yml

@@ -0,0 +1,64 @@
+# Runs the executable controls for the draft verdict evidence predicate.
+#
+# The draft (docs/predicates/acs-verdict-evidence-v1.md, #297) says in prose that a
+# scope lift must come from a statement the peer signed, and that a quarantine record
+# must be bound and must have an outcome after review_by. The controls are signed
+# records with accepted and refused cases. They come from the verdict-evidence
+# dependency group, pinned in uv.lock, so this job is the only place they install.
+# The guard job in deploy-pages.yml collects the same test file and skips it there.
+#
+# The ACS-Core suite stays in tests/ as files, per #19. This job does not change that.
+name: Verdict evidence controls
+
+on:
+  pull_request:
+    paths:
+      - "docs/predicates/**"
+      - "tests/test_verdict_evidence_controls.py"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/verdict-evidence-controls.yml"
+  push:
+    branches: ["main", "integration"]
+    paths:
+      - "docs/predicates/**"
+      - "tests/test_verdict_evidence_controls.py"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/verdict-evidence-controls.yml"
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: verdict-evidence-controls-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  controls:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # Nothing here pushes with git, so the checkout's token has no reason to
+          # stay behind in .git/config.
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          version: "0.9.9"
+
+      # --locked refuses a lockfile that drifted from pyproject.toml, so the controls
+      # that run are the ones the lockfile pins.
+      - name: Install the controls from the lockfile
+        run: uv sync --locked --group verdict-evidence
+
+      # With this set, a missing package fails the run instead of skipping it.
+      - name: Run the controls
+        env:
+          ACS_REQUIRE_VERDICT_EVIDENCE: "1"
+        run: uv run --locked --group verdict-evidence pytest -v tests/test_verdict_evidence_controls.py

pyproject.toml

@@ -12,6 +12,10 @@ dev = [ "pytest>=8.0", "jsonschema>=4.25.0",]
 # than dev, so the docs build and the test gate never install it. The lockfile pins the
 # version with hashes, and Dependabot moves it under the usual cooldown.
 zizmor = [ "zizmor>=1.30.1",]
+# The verdict evidence controls in tests/test_verdict_evidence_controls.py, run by
+# .github/workflows/verdict-evidence-controls.yml. A named group, like zizmor, so the
+# docs build and the guard job never install it. The lockfile pins the commit.
+verdict-evidence = [ "agent-evidence-vectors",]
 
 [tool.pytest.ini_options]
 # The deploy workflow runs a bare `uv run pytest -v` as the gate the site build
@@ -21,3 +25,7 @@ zizmor = [ "zizmor>=1.30.1",]
 # collection to tests/ keeps that gate about the guards it was written for.
 # Suites that carry their own dependencies run from their own workflow.
 testpaths = ["tests"]
+
+[tool.uv.sources]
+# Pinned to the commit that carries the controls until a release on PyPI does.
+agent-evidence-vectors = { git = "https://github.com/probityai/agent-evidence-vectors", rev = "617cf55151b2142c5ff4eed4aa2f4eeb5a8261c5" }

tests/test_verdict_evidence_controls.py

@@ -0,0 +1,128 @@
+"""Executable controls for the draft verdict evidence predicate (docs/predicates, #297).
+
+The draft says two things in prose. A `witness_scope` above SELF must come from a
+statement the peer authored and signed, naming the requirement it checked and the
+artifact it used, never from a field the runner writes. A quarantine record must be
+bound, and `review_by` must have an outcome. These tests run a pinned corpus of
+signed records against a reference reader. Each refused case is one change from an
+accepted twin: a wrong peer key, a peer statement the runner signed, the wrong vector,
+the wrong requirement, an absent peer artifact, a different transcript, altered
+quarantine fields and an overdue `review_by`.
+
+The corpus and the reader come from the `verdict-evidence` dependency group, pinned in
+uv.lock. The guard job does not install that group, so there the module is skipped.
+The verdict-evidence-controls workflow installs it and sets
+ACS_REQUIRE_VERDICT_EVIDENCE=1, which turns a missing package into a failure.
+
+To run a verifier of your own against the same cases, set
+ACS_VERDICT_EVIDENCE_VERIFIER to its command line. It is called as
+`<verifier> <case-dir>/case.json --json`, prints one JSON object with `decision` and
+`reason`, and exits 0 for supported, 1 for contradicted or not_established, 2 for no
+verdict.
+"""
+import importlib.util
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+REQUIRED = os.environ.get("ACS_REQUIRE_VERDICT_EVIDENCE") == "1"
+MODULE = "agent_evidence_vectors.verdict_evidence"
+
+if importlib.util.find_spec("agent_evidence_vectors") is None or (
+        importlib.util.find_spec(MODULE) is None):
+    if REQUIRED:
+        raise RuntimeError(
+            f"{MODULE} is not installed; run `uv sync --locked --group verdict-evidence`")
+    pytest.skip("the verdict-evidence dependency group is not installed",
+                allow_module_level=True)
+
+from agent_evidence_vectors.verdict_evidence import run as controls  # noqa: E402
+
+ROOT = Path(__file__).resolve().parents[1]
+PREDICATE_DOC = ROOT / "docs" / "predicates" / "acs-verdict-evidence-v1.md"
+MANIFEST = json.loads((controls.PACKAGED_CORPUS / "MANIFEST.json").read_text())
+CASES = MANIFEST["cases"]
+RUN = [sys.executable, "-m", f"{MODULE}.run"]
+
+# One refusal for each rule the draft states in prose.
+DRAFT_RULES = {
+    "peer_artifact_unverified:untrusted_key": "wrong peer key, or the runner's own key",
+    "peer_artifact_unverified:wrong_vector": "the peer checked another vector",
+    "peer_artifact_unverified:wrong_requirement": "the peer checked another requirement",
+    "peer_artifact_unverified:not_supplied": "the record names a peer artifact nobody holds",
+    "peer_artifact_unverified:transcript_digest_mismatch": "the retained transcript differs",
+    "witness_scope_misderived": "PEER or EXTERNAL with no peer statement behind it",
+    "field_invalid:provenance.lift_state": "a lift field written by the runner",
+    "peer_disagrees": "the peer saw another result, or its statement is reused for another verdict",
+    "payload_digest_fields_incomplete": "a digest that leaves out a field, such as expiry_disposition",
+    "quarantine_digest_mismatch": "a quarantine field edited after the digest",
+    "quarantine_overdue": "review_by has passed",
+}
+
+
+def _report(tmp_path, *extra):
+    out = tmp_path / "out"
+    done = subprocess.run([*RUN, "--output", str(out), *extra], capture_output=True,
+                          text=True, check=False)
+    report = json.loads((out / "report.json").read_text()) if out.exists() else None
+    return done, report
+
+
+def test_the_reference_reader_answers_every_case_as_expected(tmp_path):
+    done, report = _report(tmp_path)
+    assert done.returncode == 0, done.stdout + done.stderr
+    assert report["answered"] == report["total"] == len(CASES)
+    assert report["failures"] == []
+
+
+@pytest.mark.parametrize("reason", sorted(DRAFT_RULES))
+def test_each_rule_in_the_draft_has_a_refused_case_with_an_accepted_twin(reason):
+    by_id = {c["id"]: c for c in CASES}
+    refused = [c for c in CASES if c["expected"]["reason"] == reason]
+    assert refused, f"no case for: {DRAFT_RULES[reason]}"
+    for case in refused:
+        assert by_id[case["twin"]]["expected"]["decision"] == "supported"
+
+
+def test_no_accepted_scope_lift_rests_on_a_runner_key():
+    for case in CASES:
+        if case["expected"]["reason"] in ("verdict_verified_peer", "verdict_verified_external"):
+            spec = json.loads((controls.PACKAGED_CORPUS / "cases" / case["id"] /
+                               "case.json").read_text())
+            assert "peer_statement" in spec["supplied"]
+            assert not set(spec["policy"]["peer_keys"]) & set(spec["policy"]["trusted_keys"])
+
+
+@pytest.mark.skipif(not PREDICATE_DOC.exists(), reason="the predicate document lands with #297")
+def test_the_corpus_uses_the_predicate_type_the_document_states():
+    uris = re.findall(r"`(https://[^`]+acs-verdict-evidence-v1\.md)`", PREDICATE_DOC.read_text())
+    assert MANIFEST["verdictPredicate"] in uris
+
+
+def _stub(tmp_path, decision, code):
+    script = tmp_path / f"stub_{decision}.py"
+    script.write_text("import json, sys\n"
+                      f"print(json.dumps({{'decision': {decision!r}, 'reason': 'stub'}}))\n"
+                      f"sys.exit({code})\n")
+    return f"{shlex.quote(sys.executable)} {shlex.quote(str(script))}"
+
+
+@pytest.mark.parametrize(("decision", "code"), [("supported", 0), ("contradicted", 1)])
+def test_a_verifier_that_answers_one_way_fails_the_run(tmp_path, decision, code):
+    done, report = _report(tmp_path, "--verifier", _stub(tmp_path, decision, code))
+    assert done.returncode == 1, done.stdout + done.stderr
+    assert report["failures"], "a constant verifier must fail some case"
+
+
+@pytest.mark.skipif("ACS_VERDICT_EVIDENCE_VERIFIER" not in os.environ,
+                    reason="no verifier under test named")
+def test_the_named_verifier_answers_every_case_as_expected(tmp_path):
+    done, report = _report(tmp_path, "--verifier", os.environ["ACS_VERDICT_EVIDENCE_VERIFIER"])
+    assert done.returncode == 0, done.stdout + done.stderr
+    assert report["answered"] == report["total"]