From 4e1e0a6c7a37bf6ab66fea957a9170f2f6d77c6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:46:58 -0700 Subject: [PATCH 001/104] fix(action): fail closed on explicitly requested package version; report install origin --- action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 0bcf30906..8b4769c93 100644 --- a/action.yml +++ b/action.yml @@ -103,8 +103,14 @@ runs: pip install . } if [ -n "${OVK_PACKAGE_VERSION:-}" ]; then - pip install "open-verification-kernel==${OVK_PACKAGE_VERSION}" || install_from_checkout + # A requested distribution version is an identity constraint. + # Never silently substitute the Action checkout if PyPI resolution fails. + echo "OVK install: requested distribution version ${OVK_PACKAGE_VERSION}" + pip install "open-verification-kernel==${OVK_PACKAGE_VERSION}" + python -c 'from importlib.metadata import version; print("OVK installed distribution version:", version("open-verification-kernel"))' else + # Action source installs remain available for SHA-pinned development. + echo "OVK install: package from Action source checkout" install_from_checkout fi - name: Initialize verification directory if needed From 352bd207a8de54e044fa68001113419631084090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:47:24 -0700 Subject: [PATCH 002/104] test(action): execute install branch with fake pip to prove requested-version failures cannot fallback --- tests/test_action_package_install_contract.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 tests/test_action_package_install_contract.py diff --git a/tests/test_action_package_install_contract.py b/tests/test_action_package_install_contract.py new file mode 100644 index 000000000..37e013162 --- /dev/null +++ b/tests/test_action_package_install_contract.py @@ -0,0 +1,109 @@ +"""Composite Action installation: exact version means no silent source fallback. + +Execute the actual Action run block in a sandbox with fake commands. No +package manager or network access is needed, and failed versions cannot be +mistaken for qualified development source installs. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def _action_install_script() -> str: + data = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + install = [item for item in data["runs"]["steps"] if item.get("name") == "Install OVK package"] + assert len(install) == 1 + return install[0]["run"] + + +def _exercise(tmp_path: Path, *, requested_version: str | None, fail_requested: bool = False): + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + log_path = tmp_path / "commands.log" + for command in ("pip", "python"): + program = fake_bin / command + program.write_text( + "#!/bin/sh\n" + f'printf "{command} " >> "$OVK_TEST_LOG"\n' + 'printf "<%s>" "$@" >> "$OVK_TEST_LOG"\n' + 'printf "\\n" >> "$OVK_TEST_LOG"\n' + 'if [ "$OVK_TEST_FAIL_VERSION" = "1" ] && ' + ' [ "$1" = "install" ] && ' + ' [ "$2" = "open-verification-kernel==$OVK_PACKAGE_VERSION" ]; then\n' + ' exit 47\n' + 'fi\n' + 'exit 0\n', + encoding="utf-8", + ) + program.chmod(0o755) + env = dict(os.environ) + env.update({ + "PATH": str(fake_bin) + os.pathsep + env.get("PATH", ""), + "OVK_TEST_LOG": str(log_path), + "OVK_TEST_FAIL_VERSION": "1" if fail_requested else "0", + }) + if requested_version is None: + env.pop("OVK_PACKAGE_VERSION", None) + else: + env["OVK_PACKAGE_VERSION"] = requested_version + result = subprocess.run( + ["bash", "-c", _action_install_script()], + cwd=tmp_path, + env=env, + check=False, + text=True, + capture_output=True, + ) + return result, log_path.read_text(encoding="utf-8") + + +def test_requested_version_failed_resolution_does_not_install_from_checkout(tmp_path): + result, log = _exercise(tmp_path, requested_version="9.9.9", fail_requested=True) + assert result.returncode == 47 + assert "pip " in log + assert "sync_package_data.py" not in log + assert "pip <.>" not in log + assert "OVK installed distribution version" not in result.stdout + + +def test_requested_version_success_reports_dist_metadata_without_fallback(tmp_path): + result, log = _exercise(tmp_path, requested_version="1.2.1") + assert result.returncode == 0, result.stderr + assert "pip " in log + assert "python <-c>" in log + assert "sync_package_data.py" not in log + assert "pip <.>" not in log + assert "requested distribution version 1.2.1" in result.stdout + + +def test_unpinned_development_install_is_explicit_source_checkout(tmp_path): + result, log = _exercise(tmp_path, requested_version=None) + assert result.returncode == 0, result.stderr + assert "python " in log + assert "pip <.>" in log + assert "open-verification-kernel==" not in log + assert "package from Action source checkout" in result.stdout + + +def test_empty_version_uses_development_checkout(tmp_path): + result, log = _exercise(tmp_path, requested_version="") + assert result.returncode == 0, result.stderr + assert "pip <.>" in log + + +@pytest.mark.parametrize("supplied", ["1.2.1;touch should-not-exist", "1.2.1 $(echo injected)"]) +def test_version_is_one_quoted_package_argument(tmp_path, supplied): + result, log = _exercise(tmp_path, requested_version=supplied) + assert result.returncode == 0, result.stderr + assert "pip " in log + assert "sync_package_data.py" not in log + assert not (tmp_path / "should-not-exist").exists() From e218b71cb19c933e58c3054421dc0ac5fe42108b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:48:00 -0700 Subject: [PATCH 003/104] docs(readme): replace nonexistent RC copy-paste pin with exact published-source SHA and scope --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ddffa0027..5d913e9a1 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + # SHA of the latest published v1.2.1 release's direct source commit. + # Change this only after reviewing the release identity and limits. + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 + env: + OVK_PACKAGE_VERSION: "1.2.1" with: mode: advisory # switch to strict only after calibration use-check: "true" # analyze the PR diff automatically @@ -155,7 +159,7 @@ jobs: post-comment: "true" # requires pull-requests: write ``` -Pin target is `v1.3.0-rc.1` only after the attributable tag/release exists. Until then, do not treat that tag as a live install surface; see [docs/CURRENT_RELEASE_STATUS.md](docs/CURRENT_RELEASE_STATUS.md). +**Distribution boundary (verified 2026-10-15):** `v1.3.0-rc.1` is not published or tagged. The snippet above targets the older, published `v1.2.1` code, pinned to its exact source commit, not the current `main` development candidate or the draft guarantee-continuity feature. An explicitly set `OVK_PACKAGE_VERSION` must install that exact distribution or fail; the Action must not silently switch to checkout code. The historical `v1.2.1` tag itself is unsigned and the corresponding GitHub Release is not marked immutable. See [release status](docs/CURRENT_RELEASE_STATUS.md) and the [pilot playbook](docs/EXTERNAL_PILOT_PLAYBOOK.md) before adoption. Copy a full consumer example: [`examples/github_workflows/external_consumer.yml`](examples/github_workflows/external_consumer.yml) From e45e9102c14cc37ad2f5902cc2cc477504446f14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:48:30 -0700 Subject: [PATCH 004/104] docs(pilot): remove small-sample strict-promotion shortcut, require independent source-profile qualification --- docs/EXTERNAL_PILOT_PLAYBOOK.md | 44 +++++++++++++++------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/docs/EXTERNAL_PILOT_PLAYBOOK.md b/docs/EXTERNAL_PILOT_PLAYBOOK.md index 6f5af993a..fd57f3ce6 100644 --- a/docs/EXTERNAL_PILOT_PLAYBOOK.md +++ b/docs/EXTERNAL_PILOT_PLAYBOOK.md @@ -1,6 +1,6 @@ # External OSS Pilot Playbook -Guide for adding OVK to an external open-source repository: start in advisory mode, measure false positives, then optionally enforce on protected branches. +Guide for adding OVK to a **consenting** external open-source repository. Start in advisory mode, retain both supported and unsupported PRs, measure reviewer utility and coverage, and qualify strict enforcement only through a separate safety decision. ## Start with one check type @@ -56,15 +56,18 @@ jobs: post-comment: "false" ``` -4. Run for **two weeks** on all agent PRs without blocking merges. +4. Run for **two weeks** on the organization's natural PR stream, including relevant non-agent changes. Record all eligible and unsupported cases without blocking merges; if the sample is too small, extend the observation window and report attrition. ## Step 2 — Measure -| Metric | Target before strict | -|--------|----------------------| -| False positive rate | under 5% | -| Time to first green after repair | under 5 minutes | -| Block rate on known-bad fixtures | 100% | +| Measurement | Required reporting | +|--------|--------------------| +| Source-profile coverage | Eligible and unsupported PRs with exact denominators and known blind spots | +| Reviewer judgments | All findings reviewed, disagreements, time spent, independently discovered misses and unknown outcomes | +| Operational cost | Setup effort, per-PR runtime, reviewer minutes, repeat-check and repair work | +| Known-bad controls | All relevant admitted adversarial cases with versioned evidence; never a substitute for real-PR calibration | + +**A small pilot demonstrates feasibility, not production accuracy.** Report counts and denominators, rather than treating a threshold such as a 5% false-positive rate from a handful of PRs as sufficient for strict enforcement. Download the workflow artifact (`ovk-pilot-artifacts`) and ingest metrics into the OVK registry: @@ -77,28 +80,21 @@ Optional self-report JSON: copy [external_pilot_report.template.json](templates/ Publish results in [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md) under **External pilots**. -Maintainers ingest downloaded workflow artifacts and refresh the public summary: - -```bash -python scripts/ingest_external_pilot_metrics.py --repo org/repo --artifacts-dir ./artifact -python scripts/render_pilot_metrics.py --registry docs/benchmarks/external-pilots-registry.json -``` +Registry rows live in [external-pilots-registry.json](benchmarks/external-pilots-registry.json). Existing maintained-consumer reports are not independent customer evaluations. -Optional self-report JSON: copy [external_pilot_report.template.json](templates/external_pilot_report.template.json), fill metrics, and pass `--report external_pilot_report.json` to the ingest script. Registry rows live in [external-pilots-registry.json](benchmarks/external-pilots-registry.json). +## Step 3 — Independent strict-enforcement qualification (not an automatic upgrade) -## Step 3 — Strict on protected branches +**Do not switch a pilot to strict mode from a low measured false-positive rate alone.** Current generated source profiles are `executable_advisory`, not `externally_calibrated_strict`. -When false positives stay below 5%: +Strict enforcement must be separately authorized by the adopting organization for a *specific exact product version, supported source profile and claim*. Before using any result as a required check, document: -```yaml -- uses: fraware/open-verification-kernel@v1.2.1 - with: - mode: strict - use-check: "true" - emit-check: "true" -``` +1. Trusted acquisition of the actual repository inputs, source revision, policy and toolchain identities. +2. Source-to-model fidelity for the admitted constructs, together with explicit unsupported, unknown, timeout and error handling. +3. Human-adjudicated normal PRs and adversarial change tests, with denominators, missed issues, false alarms, and uncertainty reported. No arbitrary small-N accuracy threshold. +4. CI token permissions, adversarial fork PR isolation, bypass and workflow-self-modification resistance, protected branch rules, and a rollback owner. +5. Reviewer approval and monitoring for the exact admitted claim family. Unknown and incomplete outcomes do not count as positive verification. -Require the OVK check on `main` / `release/*` only; keep advisory on experimental branches if needed. +Until those gates are independently met, remain in **advisory mode with no merge-blocking authority**. Green fixture tests do not override this constraint. ## Step 4 — Add more check types From 9299a5e7e1a5473e02e84ba1ea5698ab1a728aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:48:55 -0700 Subject: [PATCH 005/104] docs(security): reconcile published release vs unverified maintenance commitments and private reporting --- SECURITY.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index c0a239255..ee6a1a6b5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,19 +2,22 @@ OVK is security-sensitive infrastructure because it may influence pull-request merge decisions and evidence claims. -## Supported versions +## Release and security support status -| Version | Supported | +**Last reviewed:** 2026-10-15. The latest publicly published release is `v1.2.1`. The `v1.3.0-rc.1` tag and distribution are **not published**. A historical version appearing in this table does not establish that it receives backported security fixes. + +| Version | Public release and support status | |---|---| -| 1.1.x | yes | -| 1.0.x | yes | -| < 1.0 | no | +| `1.2.1` | Latest published release; the maintainer must explicitly confirm the security-patch support commitment | +| `1.1.x`, `1.0.x` | Historical versions; earlier documentation marked these supported, but ongoing security maintenance is not independently verified | +| Before `1.0` | No support claim | +| `1.3.0-rc.1` | Unpublished engineering candidate; no public install or security-support claim | -Report security issues against `main` or the latest release tag. +Report suspected vulnerabilities affecting the published release or current source tree through a **private** channel. Do not assume that reporting a problem against an old series creates a backport commitment. ## Reporting a vulnerability -Please open a private security advisory if available, or contact the maintainers through the repository owner. +Use the repository's private vulnerability-reporting / security advisory interface if available; otherwise contact the repository maintainer privately before sharing exploit details. If no private reporting path is available, the maintainers must configure one before a wider security-sensitive launch. ## Security-critical project rules From abccfb5acff097e8cd173afa76a6941183742211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:49:14 -0700 Subject: [PATCH 006/104] docs(release): record live unsigned historical tag, missing RC ref, and absent branch protection --- docs/CURRENT_RELEASE_STATUS.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/CURRENT_RELEASE_STATUS.md b/docs/CURRENT_RELEASE_STATUS.md index 3f71a3173..cffcad2ab 100644 --- a/docs/CURRENT_RELEASE_STATUS.md +++ b/docs/CURRENT_RELEASE_STATUS.md @@ -2,9 +2,11 @@ Living release/adoption status for Open Verification Kernel. -**Last reviewed:** 2026-08-26 +**Release/ref state checked live:** 2026-10-15 (other program and maturity claims remain subject to their separate evidence). -**Current judgment:** `1.3.0-rc.1` is an **engineering release candidate after the merge of final technical-closure PR #23**. It is not yet an attributable `v1.3.0-rc.1` release. The last previously signed immutable release remains `v1.2.1`; evidence from that tag must not be re-attributed to the RC source tree. The exact RC source identity is the final post-documentation candidate SHA that passes the complete exact-head development matrix; later movement of `main` does not redefine or authorize that candidate. +**Current judgment:** `1.3.0-rc.1` is an **unpublished engineering release candidate**, not an attributable `v1.3.0-rc.1` release. On 2026-10-15, the RC tag lookup returned 404; the latest GitHub Release is `v1.2.1` (published 2026-07-23). Its annotated Git tag is reported by GitHub as **unsigned** (`verification.verified=false`, `reason=unsigned`), and its GitHub Release object reports `immutable=false`. Wheel/sdist Sigstore bundle assets are attached, but their cryptographic validity was **not independently verified by this live status review**. Do not conflate distribution signing with Git-tag signing or release immutability. + +`main` resolved to `79370539ca9b26a0a6b01f6ac6ab6923243f40f4` and reported `protected=false`; the repository rulesets list was empty. The branch-protection-detail endpoint was inaccessible to this integration (403), so administrative configuration was not inspected exhaustively. These observations do **not** establish the necessary release-ref protections. Issue #25 remains blocking. Historical artifacts from `v1.2.1` must never be re-attributed to an RC source tree. A new candidate requires the complete exact-head development matrix and independently authorized tag-bound release evidence. The authoritative publication procedure is [RELEASE.md](RELEASE.md). The evidence threshold is [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). The generated TCB is [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md). @@ -124,7 +126,7 @@ Safe recovery after a partial PyPI/GitHub failure is allowed only when existing | Strict required check | Repository-specific only after trusted inputs and empirical calibration for the intended lane | | General production-stable enforcement | **Not claimed** | -Until an attributable `v1.3.0-rc.1` release exists, do not present that tag as a live install pin. Historical `v1.2.1` remains a separate immutable release with its own source and signing evidence. +Until an attributable `v1.3.0-rc.1` release exists, do not present that tag as a live install pin. Historical `v1.2.1` is a separate **published release**, with attached signing-related assets; its **Git tag is unsigned and GitHub Release immutability is false** according to the October 15, 2026 API observation. Its distribution bundles have not been cryptographically reverified by this review. ## Remaining external/maintainer evidence for the RC From 13a8500f69b31d847699ceaab0a644af33283ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:52:45 -0700 Subject: [PATCH 007/104] docs(license): restore complete standard Apache-2.0 terms for SPDX recognition --- LICENSE | 205 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 198 insertions(+), 7 deletions(-) diff --git a/LICENSE b/LICENSE index 724b7b640..d64569567 100644 --- a/LICENSE +++ b/LICENSE @@ -1,11 +1,202 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ -Copyright 2026 fraware + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -http://www.apache.org/licenses/LICENSE-2.0 + 1. Definitions. -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 28cf5abc9cd8f1d91ad8fd00cf577d739f3cc2ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:52:52 -0700 Subject: [PATCH 008/104] docs(license): retain project copyright attribution alongside full Apache-2.0 text --- NOTICE | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..7a3e3a963 --- /dev/null +++ b/NOTICE @@ -0,0 +1,5 @@ +Open Verification Kernel + +Copyright 2026 fraware + +Licensed under the Apache License, Version 2.0. See the LICENSE file for the complete terms. From eef44e416e5da50819aa6380d66e0e50ff50964a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:53:09 -0700 Subject: [PATCH 009/104] build: include copyright NOTICE in source distribution --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8979dc07b..5c932de78 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include README.md include LICENSE +include NOTICE recursive-include schemas *.json recursive-include templates *.intent.json recursive-include adapters *.json From 9edfad78eaf3c20f60adad6457f19d7207f676ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:53:47 -0700 Subject: [PATCH 010/104] docs(readme): remove unpublished green docs badge and label checkout quickstart as development --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5d913e9a1..21e2aaff0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](pyproject.toml) [![FormalPR-Bench](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/fraware/open-verification-kernel/main/docs/benchmarks/leaderboard-badge.json)](docs/benchmarks/latest-leaderboard-summary.json) -[![Docs](https://img.shields.io/badge/docs-v1.3.0--rc.1-green.svg)](docs/README.md) +[![Docs](https://img.shields.io/badge/docs-development--candidate-informational.svg)](docs/README.md) [Quick start](#quick-start) · [Add to CI](#github-actions) · [Contribute](#contribute) · [Documentation](docs/README.md) @@ -95,10 +95,12 @@ ovk repair-suggest --evidence ovk-evidence.json --- -## Quick start +## Quick start (local development snapshot) **Requirements:** Python 3.10+ +These commands clone current, unreleased development `main`. They are for local inspection and experimentation; they do **not** install an attributable published release or qualify a strict production gate. + ```bash git clone https://github.com/fraware/open-verification-kernel.git cd open-verification-kernel From b610080bfdb5b415ca6237b377788fac2e7f7685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:53:57 -0700 Subject: [PATCH 011/104] docs(pilot): use exact historic release source SHA, not mutable tag, for consenting trials --- docs/EXTERNAL_PILOT_PLAYBOOK.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/EXTERNAL_PILOT_PLAYBOOK.md b/docs/EXTERNAL_PILOT_PLAYBOOK.md index fd57f3ce6..33a84a2c4 100644 --- a/docs/EXTERNAL_PILOT_PLAYBOOK.md +++ b/docs/EXTERNAL_PILOT_PLAYBOOK.md @@ -11,6 +11,8 @@ Guide for adding OVK to a **consenting** external open-source repository. Start Enable additional check types one at a time after each is stable on your diffs. +**Verified release distinction (2026-10-15):** the examples below pin the exact source commit of the published `v1.2.1` release; the Git tag is unsigned and its GitHub Release is not marked immutable. `OVK_PACKAGE_VERSION: 1.2.1` selects the corresponding older PyPI distribution. New guarantee-continuity reviewer work from draft #218 is **not included**. Read [current release status](CURRENT_RELEASE_STATUS.md) before installing. + ## Step 1 — Advisory mode (no merge blocking) 1. Copy `docs/templates/pilot_manifest_ci_secrets.template.json` to `.verification/ci_secrets_pilot.json` in the target repo. @@ -41,14 +43,14 @@ jobs: git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: OVK advisory check - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory use-check: "true" changed-files: ovk-pr.diff post-comment: "false" - name: OVK advisory verify (CI secrets check) - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json From 80318fafd271975b85a603f37a80515534b4fcec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:57:54 -0700 Subject: [PATCH 012/104] test(action): use shell-sourced command mocks portable across Linux, macOS and Windows Git Bash --- tests/test_action_package_install_contract.py | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/tests/test_action_package_install_contract.py b/tests/test_action_package_install_contract.py index 37e013162..ecbc6b9f8 100644 --- a/tests/test_action_package_install_contract.py +++ b/tests/test_action_package_install_contract.py @@ -8,6 +8,7 @@ from __future__ import annotations import os +import shutil import subprocess from pathlib import Path @@ -26,29 +27,37 @@ def _action_install_script() -> str: def _exercise(tmp_path: Path, *, requested_version: str | None, fail_requested: bool = False): - fake_bin = tmp_path / "bin" - fake_bin.mkdir() + # GitHub's Windows runners support Git Bash, whose PATH syntax differs + # from native Windows Python. BASH_ENV intercepts pip/python in Bash itself. + bash = shutil.which("bash") + if bash is None: + pytest.skip("Bash unavailable for POSIX composite Action execution") + (tmp_path / "mock_commands.sh").write_text( + """pip() { + printf 'pip ' >> "$OVK_TEST_LOG" + printf '<%s>' "$@" >> "$OVK_TEST_LOG" + echo >> "$OVK_TEST_LOG" + if [ "$OVK_TEST_FAIL_VERSION" = "1" ] && + [ "$1" = "install" ] && + [ "$2" = "open-verification-kernel==$OVK_PACKAGE_VERSION" ]; then + return 47 + fi + return 0 +} +python() { + printf 'python ' >> "$OVK_TEST_LOG" + printf '<%s>' "$@" >> "$OVK_TEST_LOG" + echo >> "$OVK_TEST_LOG" + return 0 +} +""", + encoding="utf-8", + ) log_path = tmp_path / "commands.log" - for command in ("pip", "python"): - program = fake_bin / command - program.write_text( - "#!/bin/sh\n" - f'printf "{command} " >> "$OVK_TEST_LOG"\n' - 'printf "<%s>" "$@" >> "$OVK_TEST_LOG"\n' - 'printf "\\n" >> "$OVK_TEST_LOG"\n' - 'if [ "$OVK_TEST_FAIL_VERSION" = "1" ] && ' - ' [ "$1" = "install" ] && ' - ' [ "$2" = "open-verification-kernel==$OVK_PACKAGE_VERSION" ]; then\n' - ' exit 47\n' - 'fi\n' - 'exit 0\n', - encoding="utf-8", - ) - program.chmod(0o755) env = dict(os.environ) env.update({ - "PATH": str(fake_bin) + os.pathsep + env.get("PATH", ""), - "OVK_TEST_LOG": str(log_path), + "BASH_ENV": "./mock_commands.sh", + "OVK_TEST_LOG": "commands.log", "OVK_TEST_FAIL_VERSION": "1" if fail_requested else "0", }) if requested_version is None: @@ -56,7 +65,7 @@ def _exercise(tmp_path: Path, *, requested_version: str | None, fail_requested: else: env["OVK_PACKAGE_VERSION"] = requested_version result = subprocess.run( - ["bash", "-c", _action_install_script()], + [bash, "-c", _action_install_script()], cwd=tmp_path, env=env, check=False, From c613098ed0ced8a252a89cb81224826581e3f87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:05:32 -0700 Subject: [PATCH 013/104] docs(readme): disclose actual legacy Action fallback; reserve fail-closed claim for unreleased change --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21e2aaff0..b6f27de27 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ jobs: post-comment: "true" # requires pull-requests: write ``` -**Distribution boundary (verified 2026-10-15):** `v1.3.0-rc.1` is not published or tagged. The snippet above targets the older, published `v1.2.1` code, pinned to its exact source commit, not the current `main` development candidate or the draft guarantee-continuity feature. An explicitly set `OVK_PACKAGE_VERSION` must install that exact distribution or fail; the Action must not silently switch to checkout code. The historical `v1.2.1` tag itself is unsigned and the corresponding GitHub Release is not marked immutable. See [release status](docs/CURRENT_RELEASE_STATUS.md) and the [pilot playbook](docs/EXTERNAL_PILOT_PLAYBOOK.md) before adoption. +**Release safety boundary (verified 2026-10-15):** `v1.3.0-rc.1` is not published or tagged. This example runs **historical `v1.2.1` code**, pinned to its original source commit; it does not include the draft guarantee-continuity feature or the new fail-closed installer. **The `v1.2.1` Action falls back to its checkout if the requested PyPI version fails to install.** Do not treat that installation as independently authenticated solely because `OVK_PACKAGE_VERSION` is set. The old Git tag is unsigned, and its GitHub Release is not marked immutable. The corrected installation contract in draft PR #220 takes effect only in a later, independently authorized release. Until then use the checkout examples for development or a separately governed, consenting **advisory-only** experiment. See [release status](docs/CURRENT_RELEASE_STATUS.md), [pilot playbook](docs/EXTERNAL_PILOT_PLAYBOOK.md), and [launch checklist](docs/DEVELOPER_PREVIEW_READINESS.md). Copy a full consumer example: [`examples/github_workflows/external_consumer.yml`](examples/github_workflows/external_consumer.yml) From a9e6acfe4c346f3b80f217528ff8b48a5c49608e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:06:16 -0700 Subject: [PATCH 014/104] docs(pilot): replace unpublished RC pins; disclose legacy installer and fork permissions --- examples/github_workflows/pilot_fork_adopter.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/github_workflows/pilot_fork_adopter.yml b/examples/github_workflows/pilot_fork_adopter.yml index f1dea0edf..aca8eb42a 100644 --- a/examples/github_workflows/pilot_fork_adopter.yml +++ b/examples/github_workflows/pilot_fork_adopter.yml @@ -10,11 +10,12 @@ on: permissions: contents: read - pull-requests: write env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + # Historical v1.2.1 distribution. This old Action has a fallback to checkout + # if package installation fails; pinning does not establish package-byte identity. + # For consenting advisory evaluations only; see release status before use. + OVK_PACKAGE_VERSION: "1.2.1" jobs: ovk-ci-secrets-pilot: @@ -28,19 +29,19 @@ jobs: git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: OVK advisory check - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory use-check: "true" changed-files: ovk-pr.diff post-comment: "false" - name: OVK advisory verify (ci_secrets lane manifest) - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json bundle-output-dir: ovk-pilot-bundle - post-comment: "true" + post-comment: "false" - name: Upload pilot metrics artifacts uses: actions/upload-artifact@v4 with: From 352ff603704ea22e229ecd2248156ffe05565c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:06:19 -0700 Subject: [PATCH 015/104] docs(pilot): replace unpublished RC pins; disclose legacy installer and fork permissions --- .../github_workflows/pilot_advisory_with_comment.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/github_workflows/pilot_advisory_with_comment.yml b/examples/github_workflows/pilot_advisory_with_comment.yml index 6523b0324..cf8298b05 100644 --- a/examples/github_workflows/pilot_advisory_with_comment.yml +++ b/examples/github_workflows/pilot_advisory_with_comment.yml @@ -1,6 +1,8 @@ name: OVK Pilot (Advisory + Check) # Phase 1: advisory mode with GitHub check observability (no merge blocking). +# For fork PRs GitHub normally issues read-only tokens; write operations are +# best-effort. Set post-comment: false for untrusted forks. # Copy to .github/workflows/ovk-advisory-check.yml in your repository. on: @@ -13,8 +15,10 @@ permissions: checks: write env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + # Historical v1.2.1 distribution. This old Action has a fallback to checkout + # if package installation fails; pinning does not establish package-byte identity. + # For consenting advisory evaluations only; see release status before use. + OVK_PACKAGE_VERSION: "1.2.1" jobs: ovk-advisory-check: @@ -22,7 +26,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: OVK advisory check with emitted GitHub check - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory use-check: "true" From 7275fead5182d229a822a19d7f87df686bb6d3ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:06:38 -0700 Subject: [PATCH 016/104] docs(integration): distinguish historical Action fallback from future fail-closed release --- docs/INTEGRATION.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index c36591925..134b9dcd6 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -24,7 +24,7 @@ pip install open-verification-kernel==1.3.0-rc.1 pip install "open-verification-kernel[solvers]==1.3.0-rc.1" ``` -Until the wheel is on PyPI, use `pip install -e '.[dev]'` from a checkout. **Live signed pin today:** `@v1.2.1` with `OVK_PACKAGE_VERSION: "1.2.1"`. **Candidate after attributable tag:** `@v1.3.0-rc.1` with matching `OVK_PACKAGE_VERSION`. Do not pin `@main` or treat uncommitted branch work as a verified release. +For current, **unreleased development code**, use `pip install -e '.[dev]'` from a checkout. The last published GitHub Release is `v1.2.1`; its annotated Git tag is unsigned, its GitHub Release is not marked immutable, and its historical Action **falls back to checkout code if the requested package version cannot be installed**. Its exact source commit is `a27d5720f4350c00bca34f71d991c31f5a2f38c7`. An attributable `v1.3.0-rc.1` release is **not currently available**. The commands in the preceding PyPI section are forward-looking and must not be run as current-release instructions. The fail-closed installer introduced in draft PR #220 will apply only to a later independently authorized release. Optional Z3: `pip install -e '.[solvers]'` @@ -91,6 +91,8 @@ permissions: **Third-party action pins:** `action.yml` and `.github/workflows/publish.yml` pin `actions/*` (and other release third-parties) to immutable commit SHAs. CI enforces this with `python scripts/pin_action_shas.py`. +**Historical example warning:** the Action snippets below use the exact `v1.2.1` source commit for reproducibility of the *GitHub source reference*, not an authorized new release. That Action still has an installation fallback. Their configurations are **advisory/legacy illustrations only**, not a claim of fully authenticated package-byte identity. Do not promote to a required strict check solely from this example. + ### Single check type (self-protection) ```yaml @@ -107,7 +109,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: fraware/open-verification-kernel@v1.2.1 + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 id: ovk with: mode: advisory @@ -125,7 +127,7 @@ Development reference: `uses: ./` ### All five check types (manifest) ```yaml -- uses: fraware/open-verification-kernel@v1.2.1 +- uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory verification-manifest: .verification/full_mvp.json @@ -161,7 +163,7 @@ When `verification-manifest` is set, the manifest path wins over `use-check`. Downstream example: ```yaml -- uses: fraware/open-verification-kernel@v1.2.1 +- uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 id: ovk with: mode: strict @@ -217,7 +219,7 @@ Explicit metadata (most reliable): ``` ```yaml -- uses: fraware/open-verification-kernel@v1.2.1 +- uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: check-metadata: ovk-required-checks.json ``` @@ -285,7 +287,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: fraware/open-verification-kernel@v1.2.1 + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 env: OVK_PACKAGE_VERSION: "1.2.1" with: From adfd938ff86d9860451f779d2917d4854765eb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:07:35 -0700 Subject: [PATCH 017/104] docs(launch): define reproducible first-user task, explicit release gate and organic-adoption metrics --- docs/DEVELOPER_PREVIEW_READINESS.md | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 docs/DEVELOPER_PREVIEW_READINESS.md diff --git a/docs/DEVELOPER_PREVIEW_READINESS.md b/docs/DEVELOPER_PREVIEW_READINESS.md new file mode 100644 index 000000000..b78f0c5b2 --- /dev/null +++ b/docs/DEVELOPER_PREVIEW_READINESS.md @@ -0,0 +1,59 @@ +# Developer preview readiness and first-user activation + +**Status: NOT LAUNCH-AUTHORIZED.** This checklist is for the maintainer and an independent release reviewer. Do not present green PR CI or a draft PR as a published, signed production version. + +## The promise to test + +For one supported change in a real PR, the developer receives a reviewable machine-readable result that identifies (a) the bounded claim checked, (b) what evidence supports the outcome and the relevant source revision, (c) unsupported/unknown conditions, and (d) the exact next verification action. The objective is to make security-sensitive AI-authored changes reviewable with accountable evidence. Do not call an entire repository or PR formally verified. + +**Distribution strategy:** open-source CLI and composite GitHub Action as the first developer-native interface; shareable PR reports once their source revision and trust scope are reliably bound; GitHub App and optional organization service only following repeated external demand. A standalone hosted web application is not a release prerequisite. + +## First local demonstration (unreleased checkout only) + +The following runs against the **source checkout**, not a published distribution. Use Python 3.10–3.12 in a clean environment: + +```bash +git clone https://github.com/fraware/open-verification-kernel.git +cd open-verification-kernel +python -m pip install -e '.[dev]' +ovk init +ovk doctor +ovk check --changed-files examples/multi_surface/pr_combined.diff --advisory +``` + +Inspect the generated `ovk-evidence.json`, `ovk-pr-comment.md`, and `ovk-evidence-quality.json`. These are **bounded development fixtures** for product understanding, not independent accuracy evidence. Try the deterministic failing/repair walkthrough with: + +```bash +python examples/repair_loops/ci_secrets/demo_repair_loop.py +``` + +The emitted outputs must be explained in the context of the admitted inputs and compiler/checker assumptions. In advisory mode the job can succeed while the **recorded decision** remains `block`, `needs_review`, or `unknown`. + +## Historical release versus current development + +As checked October 15, 2026, `v1.2.1` is the last public GitHub Release; `v1.3.0-rc.1` **does not exist as a public tag**. The historical source commit for `v1.2.1` is `a27d5720f4350c00bca34f71d991c31f5a2f38c7`, whose **old Action implementation silently falls back from failed versioned PyPI installation to its checkout**. Its annotated tag is unsigned and the Release object is not marked immutable. This is insufficient to claim an authenticated, fail-closed distribution solely from the Action reference. + +Draft [PR #220](https://github.com/fraware/open-verification-kernel/pull/220) corrects the current checkout's installer; it does **not** retroactively patch the old Action. Do not advertise a corrected release until it is independently authorized and published. + +## Release GO / NO-GO acceptance criteria + +- [ ] GitHub administrator creates active branch and tag rulesets and the effective constraints are independently read back. See [issue #25](https://github.com/fraware/open-verification-kernel/issues/25). +- [ ] The complete exact SHA candidate passes CI, multi-OS/Python reproducibility, native backend matrices, package build, outside-checkout smoke, templates, and Action dogfood. +- [ ] The requested distribution version either installs exactly or the installation fails, with no silent checkout fallback; negative test runs on each supported platform. +- [ ] Final README, example workflows, security support policy, attribution, and license are consistent with the publication state. No nonexistent tag is in **active** copyable examples. +- [ ] The signed annotated release tag directly identifies the authorized source SHA and GitHub reports verified signature; new tag-bound release, holdout, consumer, and ledger checks meet [RELEASE.md](RELEASE.md). +- [ ] Sigstore material is cryptographically verified against exact published distribution hashes, and PyPI read-back equals the authorized bytes. Report historical artifact-signing and tag-signing claims separately. +- [ ] One external maintainer explicitly consents to a **nonblocking advisory** pilot with their normal PR stream. A reviewer inspects at least one real supported/unsupported change, recording denominators, unknowns and review time. Small samples cannot qualify strict enforcement. +- [ ] A new developer outside the maintainer team completes the **source checkout / released-wheel** first-run task and explains one result, without private fixture knowledge or maintainer intervention. Record errors and elapsed setup time. + +All boxes are independent; the absence of any source/provenance or policy evidence prohibits broad claims. The presence of published code does not grant automatic merge authority. + +## Adoption and repeat-use metrics + +Measure the conversion sequence **README viewed → independently installed → first meaningful report → second natural PR reviewed → teammate adoption**. Exclude bots and the maintainer's own controlled repositories from independent organization counts. Track unsupported source coverage as prominently as supported paths. Record reviewer time and avoided rechecking separately from compute duration; do not infer a breach-prevention rate from a tiny sample. + +**Primary North Star metric:** independently reviewed, consequential software guarantees that remain justified across successive changes, with observed revalidation effort and explicit unknowns. Stars, website impressions, and reposts are secondary distribution signals, not proof of customer value. + +## Expansion decision + +A public GitHub App follows repeated Action use **and** demonstrated onboarding friction that an App meaningfully reduces. A hosted evidence history follows paid demand for multi-repository persistence, access control, revocation, audit policy, and operational support. Public viral marketing follows a real consented demonstration with exact version-pinned, replayable materials and carefully stated guarantees. From bb2919f2ac6405bf3e1dcb94e457323718a80491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:08:20 -0700 Subject: [PATCH 018/104] test(launch): guard public pins, release scope and standard licensing --- tests/test_adoption_documentation_contract.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/test_adoption_documentation_contract.py diff --git a/tests/test_adoption_documentation_contract.py b/tests/test_adoption_documentation_contract.py new file mode 100644 index 000000000..5674e2e98 --- /dev/null +++ b/tests/test_adoption_documentation_contract.py @@ -0,0 +1,80 @@ +"""Public onboarding must not advertise an unpublished or unauthenticated release. + +The Git tag, Action source and Python distribution are distinct objects; +these tests guard copyable guidance, not the live publication authority. +""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HISTORICAL_SOURCE = "a27d5720f4350c00bca34f71d991c31f5a2f38c7" +UNPUBLISHED_TAG = "fraware/open-verification-kernel@v1.3.0-rc.1" +EXAMPLES = [ + ROOT / "examples/github_workflows/pilot_fork_adopter.yml", + ROOT / "examples/github_workflows/pilot_advisory_with_comment.yml", +] + + +def test_public_readme_marks_published_source_and_legacy_fallback() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + assert HISTORICAL_SOURCE in readme + assert 'OVK_PACKAGE_VERSION: "1.2.1"' in readme + assert "v1.3.0-rc.1" in readme + assert "not published or tagged" in readme + assert "Action falls back to its checkout" in readme + assert "fail-closed installer" in readme + assert UNPUBLISHED_TAG not in readme + + +def test_active_pilot_workflows_do_not_install_nonexistent_candidate() -> None: + for path in EXAMPLES: + text = path.read_text(encoding="utf-8") + assert UNPUBLISHED_TAG not in text, path + assert 'OVK_PACKAGE_VERSION: "1.2.1"' in text, path + assert f"fraware/open-verification-kernel@{HISTORICAL_SOURCE}" in text, path + assert "falls back to checkout" in text, path + + +def test_fork_example_does_not_request_write_permissions_or_comment() -> None: + text = EXAMPLES[0].read_text(encoding="utf-8") + assert "pull-requests: write" not in text + assert "checks: write" not in text + assert 'post-comment: "true"' not in text + + +def test_integration_guide_separates_legacy_action_and_future_release() -> None: + text = (ROOT / "docs/INTEGRATION.md").read_text(encoding="utf-8") + assert "historical Action" in text + assert "not currently available" in text + assert HISTORICAL_SOURCE in text + assert UNPUBLISHED_TAG not in text + + +def test_full_license_and_notice_are_included_in_sdist() -> None: + license_text = (ROOT / "LICENSE").read_text(encoding="utf-8") + assert "Apache License" in license_text + assert "TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION" in license_text + assert "END OF TERMS AND CONDITIONS" in license_text + assert "Copyright 2026 fraware" in (ROOT / "NOTICE").read_text(encoding="utf-8") + assert "include NOTICE" in (ROOT / "MANIFEST.in").read_text(encoding="utf-8") + + +def test_preview_checklist_and_current_status_do_not_grant_release_authority() -> None: + checklist = (ROOT / "docs/DEVELOPER_PREVIEW_READINESS.md").read_text(encoding="utf-8") + status = (ROOT / "docs/CURRENT_RELEASE_STATUS.md").read_text(encoding="utf-8") + assert "NOT LAUNCH-AUTHORIZED" in checklist + assert "no public tag" in checklist + assert "unsigned" in status + assert "immutable=false" in status + assert "protected=false" in status + assert "issue #25" in status + + +def test_no_error_swallowing_for_requested_action_package_version() -> None: + action = (ROOT / "action.yml").read_text(encoding="utf-8") + requested = 'pip install "open-verification-kernel==$' + '{OVK_PACKAGE_VERSION}"' + assert requested in action + assert requested + " ||" not in action + assert 'echo "OVK install: requested distribution version' in action From 691d31ed456ebc6c3d3f0ee5cb8d84c7bc662f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:08:44 -0700 Subject: [PATCH 019/104] test(launch): match exact legacy fallback and unpublished-tag language --- tests/test_adoption_documentation_contract.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_adoption_documentation_contract.py b/tests/test_adoption_documentation_contract.py index 5674e2e98..fee7c45d3 100644 --- a/tests/test_adoption_documentation_contract.py +++ b/tests/test_adoption_documentation_contract.py @@ -34,7 +34,7 @@ def test_active_pilot_workflows_do_not_install_nonexistent_candidate() -> None: assert UNPUBLISHED_TAG not in text, path assert 'OVK_PACKAGE_VERSION: "1.2.1"' in text, path assert f"fraware/open-verification-kernel@{HISTORICAL_SOURCE}" in text, path - assert "falls back to checkout" in text, path + assert "fallback to checkout" in text, path def test_fork_example_does_not_request_write_permissions_or_comment() -> None: @@ -65,7 +65,7 @@ def test_preview_checklist_and_current_status_do_not_grant_release_authority() - checklist = (ROOT / "docs/DEVELOPER_PREVIEW_READINESS.md").read_text(encoding="utf-8") status = (ROOT / "docs/CURRENT_RELEASE_STATUS.md").read_text(encoding="utf-8") assert "NOT LAUNCH-AUTHORIZED" in checklist - assert "no public tag" in checklist + assert "not exist as a public tag" in checklist assert "unsigned" in status assert "immutable=false" in status assert "protected=false" in status From 5dc984c1eb3b4cea8b66ab1254116197e5cec500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:09:16 -0700 Subject: [PATCH 020/104] test(action): resolve Git Bash explicitly on Windows runners before skipping --- tests/test_action_package_install_contract.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_action_package_install_contract.py b/tests/test_action_package_install_contract.py index ecbc6b9f8..80d3c7b90 100644 --- a/tests/test_action_package_install_contract.py +++ b/tests/test_action_package_install_contract.py @@ -30,6 +30,15 @@ def _exercise(tmp_path: Path, *, requested_version: str | None, fail_requested: # GitHub's Windows runners support Git Bash, whose PATH syntax differs # from native Windows Python. BASH_ENV intercepts pip/python in Bash itself. bash = shutil.which("bash") + if bash is None and os.name == "nt": + program_files = Path(os.environ.get("ProgramFiles", "C:/Program Files")) + for candidate in ( + program_files / "Git" / "bin" / "bash.exe", + program_files / "Git" / "usr" / "bin" / "bash.exe", + ): + if candidate.is_file(): + bash = str(candidate) + break if bash is None: pytest.skip("Bash unavailable for POSIX composite Action execution") (tmp_path / "mock_commands.sh").write_text( From a7249d4dcfd7e6b970b84ac9469921d79a7d9f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:11:06 -0700 Subject: [PATCH 021/104] docs(onboarding): replace non-installable strict example with least-privilege advisory workflow --- .../github_workflows/external_consumer.yml | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/examples/github_workflows/external_consumer.yml b/examples/github_workflows/external_consumer.yml index 572bbebb8..c76f21362 100644 --- a/examples/github_workflows/external_consumer.yml +++ b/examples/github_workflows/external_consumer.yml @@ -1,7 +1,12 @@ -name: OVK External Consumer Example +name: OVK External Advisory Consumer -# Phase 2: strict enforcement with emit-check for production adoption. -# See also: pilot_fork_adopter.yml (phase 0), pilot_advisory_with_comment.yml (phase 1). +# Minimum-permission demonstration for a consenting engineering team. +# Copy to .github/workflows/ovk-advisory.yml and keep non-blocking. +# Historical v1.2.1 installs can fall back to the Action checkout if the +# requested package cannot be installed. The exact Action source SHA is +# pinned, but this old version does NOT supply a fail-closed package identity. +# The new contract in draft PR #220 is unpublished; see release status. +# Do not make this advisory check a required merge gate. on: pull_request: @@ -9,35 +14,23 @@ on: permissions: contents: read - pull-requests: write - checks: write env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + OVK_PACKAGE_VERSION: "1.2.1" jobs: - ovk-check: + ovk-advisory: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Run Open Verification Kernel check path - uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: - mode: strict - use-check: "true" - emit-check: "true" - changed-files: examples/ci_secrets/workflow_secrets_on_pr.diff - post-comment: "false" - - ovk-full-mvp: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run Open Verification Kernel (all five lanes) - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + persist-credentials: false + - name: Run bounded OVK PR checks + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory - verification-manifest: examples/verification_manifests/full_mvp.json - bundle-output-dir: ovk-full-mvp-bundle + use-check: "true" + emit-check: "false" post-comment: "false" + # The Action uploads structured evidence artifacts. A green job in + # advisory mode is not proof that the underlying result is an allow. From 37c9d81b7aa71ba2d2f2729583e1fb8923034e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:11:16 -0700 Subject: [PATCH 022/104] docs(strict): make required-check template explicitly noninstallable until qualified --- .../branch_protection_required_check.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/github_workflows/branch_protection_required_check.yml b/examples/github_workflows/branch_protection_required_check.yml index 76e2668c5..6edbca77d 100644 --- a/examples/github_workflows/branch_protection_required_check.yml +++ b/examples/github_workflows/branch_protection_required_check.yml @@ -1,6 +1,10 @@ name: OVK Branch Protection (Strict Required Check) -# Phase 2+: strict enforcement with required check "Open Verification Kernel". +# FUTURE REFERENCE ONLY — NOT INSTALLABLE UNTIL REPLACED AND AUTHORIZED. +# Strict enforcement is prohibited for current executable_advisory source profiles. +# Only qualify this after independent external calibration of the exact claim +# family, protected source acquisition, live policy, authorized package and +# a repository-owner decision. Do not copy this template into active CI. # Add the check name to branch protection after this workflow runs green on a sample PR. on: @@ -12,8 +16,8 @@ permissions: checks: write env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + # Placeholder, deliberately not a published/working version. + OVK_PACKAGE_VERSION: "REPLACE_WITH_AUTHORIZED_VERSION" jobs: ovk-required-check: @@ -27,7 +31,7 @@ jobs: git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: Run OVK strict check (required) - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@REPLACE_WITH_VERIFIED_SHA with: mode: strict use-check: "true" From cc81702263ffab0fbfcb6044c67d470c44c950c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:11:29 -0700 Subject: [PATCH 023/104] docs(adoption): align copyable example and integration guide with non-blocking least privilege --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6f27de27..15bbd5b5b 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ jobs: **Release safety boundary (verified 2026-10-15):** `v1.3.0-rc.1` is not published or tagged. This example runs **historical `v1.2.1` code**, pinned to its original source commit; it does not include the draft guarantee-continuity feature or the new fail-closed installer. **The `v1.2.1` Action falls back to its checkout if the requested PyPI version fails to install.** Do not treat that installation as independently authenticated solely because `OVK_PACKAGE_VERSION` is set. The old Git tag is unsigned, and its GitHub Release is not marked immutable. The corrected installation contract in draft PR #220 takes effect only in a later, independently authorized release. Until then use the checkout examples for development or a separately governed, consenting **advisory-only** experiment. See [release status](docs/CURRENT_RELEASE_STATUS.md), [pilot playbook](docs/EXTERNAL_PILOT_PLAYBOOK.md), and [launch checklist](docs/DEVELOPER_PREVIEW_READINESS.md). -Copy a full consumer example: [`examples/github_workflows/external_consumer.yml`](examples/github_workflows/external_consumer.yml) +Copy the **minimal advisory-only** consumer example: [`examples/github_workflows/external_consumer.yml`](examples/github_workflows/external_consumer.yml). It uses the historical Action and discloses its fallback limitation; it does not enable strict enforcement or require write tokens. --- From 5e32ad6d0dadd69ab7589b847ef5610c36095ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:11:32 -0700 Subject: [PATCH 024/104] docs(adoption): align copyable example and integration guide with non-blocking least privilege --- docs/INTEGRATION.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index 134b9dcd6..3e4516ff2 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -124,6 +124,8 @@ jobs: Development reference: `uses: ./` +The recommended copyable external workflow is [the non-blocking, read-only advisory example](../examples/github_workflows/external_consumer.yml). Required strict checks are not qualified for the current source profiles. + ### All five check types (manifest) ```yaml @@ -166,7 +168,7 @@ Downstream example: - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 id: ovk with: - mode: strict + mode: advisory use-check: "true" emit-check: "true" - name: Fail when OVK blocks From c2c6c0a632b161cd98f2b0f8f68cef4a066cb0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:11:34 -0700 Subject: [PATCH 025/104] docs(adoption): align copyable example and integration guide with non-blocking least privilege --- docs/EXTERNAL_PILOT_PLAYBOOK.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/EXTERNAL_PILOT_PLAYBOOK.md b/docs/EXTERNAL_PILOT_PLAYBOOK.md index 33a84a2c4..7c1942602 100644 --- a/docs/EXTERNAL_PILOT_PLAYBOOK.md +++ b/docs/EXTERNAL_PILOT_PLAYBOOK.md @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - pull-requests: write + # No token write permissions for fork-compatible advisory evaluation. steps: - uses: actions/checkout@v4 with: From 525a854df31c604215c8ea2c5f96654f609c8ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:11:49 -0700 Subject: [PATCH 026/104] test(onboarding): guard least-privilege advisory installation and inert strict placeholder --- tests/test_adoption_documentation_contract.py | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/test_adoption_documentation_contract.py b/tests/test_adoption_documentation_contract.py index fee7c45d3..aa85a36ca 100644 --- a/tests/test_adoption_documentation_contract.py +++ b/tests/test_adoption_documentation_contract.py @@ -14,6 +14,7 @@ EXAMPLES = [ ROOT / "examples/github_workflows/pilot_fork_adopter.yml", ROOT / "examples/github_workflows/pilot_advisory_with_comment.yml", + ROOT / "examples/github_workflows/external_consumer.yml", ] @@ -34,7 +35,9 @@ def test_active_pilot_workflows_do_not_install_nonexistent_candidate() -> None: assert UNPUBLISHED_TAG not in text, path assert 'OVK_PACKAGE_VERSION: "1.2.1"' in text, path assert f"fraware/open-verification-kernel@{HISTORICAL_SOURCE}" in text, path - assert "fallback to checkout" in text, path + assert ( + "fallback to checkout" in text or "fall back to the Action checkout" in text + ), path def test_fork_example_does_not_request_write_permissions_or_comment() -> None: @@ -78,3 +81,27 @@ def test_no_error_swallowing_for_requested_action_package_version() -> None: assert requested in action assert requested + " ||" not in action assert 'echo "OVK install: requested distribution version' in action + + +def test_copyable_external_consumer_is_advisory_and_least_privilege() -> None: + text = (ROOT / "examples/github_workflows/external_consumer.yml").read_text( + encoding="utf-8" + ) + assert "mode: advisory" in text + assert "mode: strict" not in text + assert "contents: read" in text + assert "checks: write" not in text + assert "pull-requests: write" not in text + assert 'emit-check: "false"' in text + assert 'post-comment: "false"' in text + assert "changed-files:" not in text + + +def test_strict_required_check_reference_not_copyable_as_unpublished_rc() -> None: + text = ( + ROOT / "examples/github_workflows/branch_protection_required_check.yml" + ).read_text(encoding="utf-8") + assert UNPUBLISHED_TAG not in text + assert "REPLACE_WITH_AUTHORIZED_VERSION" in text + assert "REPLACE_WITH_VERIFIED_SHA" in text + assert "NOT INSTALLABLE UNTIL REPLACED" in text From 56c5f292efdae282b48b36412cae7658cea07720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:30 -0700 Subject: [PATCH 027/104] integration(action): retain fail-closed distribution pin and bind PR diff to merge preview --- action.yml | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/action.yml b/action.yml index 8b4769c93..b241e3f4f 100644 --- a/action.yml +++ b/action.yml @@ -131,40 +131,26 @@ runs: shell: bash working-directory: ${{ github.workspace }} env: + OVK_EVENT_NAME: ${{ github.event_name }} OVK_CHANGED_FILES: ${{ inputs.changed-files }} + OVK_MODE: ${{ inputs.mode }} OVK_EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} OVK_EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} OVK_GITHUB_SHA: ${{ github.sha }} run: | set -euo pipefail - if [ -n "$OVK_CHANGED_FILES" ]; then - if [ ! -f "$OVK_CHANGED_FILES" ]; then - echo "OVK changed-files input does not exist: $OVK_CHANGED_FILES" >&2 - exit 1 - fi - printf '%s\n' "$OVK_CHANGED_FILES" > .ovk-effective-input-path - exit 0 - fi - - HEAD_SHA="${OVK_EVENT_HEAD_SHA:-$OVK_GITHUB_SHA}" - BASE_SHA="${OVK_EVENT_BASE_SHA:-}" - if [ -n "$BASE_SHA" ] && ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then - git fetch --no-tags --depth=1 origin "$BASE_SHA" || true - fi - if [ -n "$HEAD_SHA" ] && ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then - git fetch --no-tags --depth=1 origin "$HEAD_SHA" || true - fi - if [ -n "$BASE_SHA" ] && git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null \ - && [ -n "$HEAD_SHA" ] && git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then - git diff --no-ext-diff --binary "$BASE_SHA" "$HEAD_SHA" > .ovk-pr.diff - elif [ -n "$BASE_SHA" ] && git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then - git diff --no-ext-diff --binary "$BASE_SHA" HEAD > .ovk-pr.diff - elif git rev-parse HEAD^ >/dev/null 2>&1; then - git diff --no-ext-diff --binary HEAD^ HEAD > .ovk-pr.diff - else - git show --format= --no-ext-diff --binary HEAD > .ovk-pr.diff - fi - printf '%s\n' ".ovk-pr.diff" > .ovk-effective-input-path + python "${{ github.action_path }}/scripts/prepare_verified_change_input.py" \ + --event-name "$OVK_EVENT_NAME" \ + --github-sha "$OVK_GITHUB_SHA" \ + --base-sha "$OVK_EVENT_BASE_SHA" \ + --pr-head-sha "$OVK_EVENT_HEAD_SHA" \ + --changed-files "$OVK_CHANGED_FILES" \ + --mode "$OVK_MODE" + - name: Disclose unbound caller-selected inputs + if: ${{ inputs.changed-files != '' && inputs.verification-manifest == '' }} + shell: bash + run: | + echo '::warning::OVK caller-supplied change input is not bound to the checked-out source; automated GitHub Check/comment emission is disabled.' - name: Run OVK multi-lane verification id: verify if: ${{ inputs.verification-manifest != '' }} @@ -239,7 +225,7 @@ runs: echo "$ACTION_RC" > .ovk-action-exit-code - name: Emit OVK GitHub check (check path) id: emit-check-run - if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest == '' && inputs.use-check == 'true' }} + if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest == '' && inputs.use-check == 'true' && inputs.changed-files == '' }} shell: bash working-directory: ${{ github.workspace }} env: @@ -335,7 +321,7 @@ runs: set -e echo "$ACTION_RC" > .ovk-action-exit-code - name: Post OVK pull-request comment - if: ${{ inputs.post-comment == 'true' }} + if: ${{ inputs.post-comment == 'true' && (inputs.changed-files == '' || inputs.verification-manifest != '') }} shell: bash working-directory: ${{ github.workspace }} env: @@ -459,4 +445,5 @@ runs: ${{ inputs.bundle-output-dir }}/** ovk-branch-metadata.json .ovk-pr.diff + ovk-source-binding.json .ovk-check-emitted From 04cd26f6a6cd15b59608e009ac918e6a389974f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:38 -0700 Subject: [PATCH 028/104] integration(action): import qualified source-binding .github/workflows/ci.yml --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc2349af1..d79e20523 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,27 +188,28 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - checks: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} + # GitHub's pull_request github.sha is the synthetic merge preview. + ref: ${{ github.sha }} + fetch-depth: 2 persist-credentials: false - - name: Strict emit-check on known-bad diff + - name: Strict Action must reject caller-selected unbound input id: ovk continue-on-error: true uses: ./ with: mode: strict use-check: "true" - emit-check: "true" + emit-check: "false" changed-files: examples/ci_secrets/workflow_secrets_on_pr.diff post-comment: "false" - - name: Assert strict block and observable check + - name: Assert unsafe input refused before verification run: | test "${{ steps.ovk.outcome }}" = "failure" - test "${{ steps.ovk.outputs.recommendation }}" = "block" - test "${{ steps.ovk.outputs.exit_code }}" = "1" + test ! -f ovk-source-binding.json + test ! -f ovk-evidence.json action_default_diff_dogfood: runs-on: ubuntu-latest @@ -217,7 +218,8 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} + # Match the actual GitHub event checkout subject, not PR-head. + ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false - name: Exercise automatic diff collection @@ -231,6 +233,20 @@ jobs: run: | test -f ovk-evidence.json test -f ovk-evidence-quality.json + test -f ovk-source-binding.json + python - <<'PY' + import json + import os + from pathlib import Path + r = json.loads(Path("ovk-source-binding.json").read_text()) + assert r["evaluated_checkout_commit_sha"] == os.environ["GITHUB_SHA"] + assert r["evidence_subject_sha"] == os.environ["GITHUB_SHA"] + if os.environ["GITHUB_EVENT_NAME"] == "pull_request": + assert r["input_comparison"] == "event_base_to_verified_merge_preview" + assert r["input_structurally_bound_to_checkout"] is True + assert r["bound_into_signed_evidence_envelope"] is False + assert r["production_reliance_authorized"] is False + PY ovk evidence-quality ovk-evidence.json package: From 9a776e6fbae849b5cad8da6be4a52d02926a675d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:40 -0700 Subject: [PATCH 029/104] integration(action): import qualified source-binding tests/test_release_workflow_event_provenance.py --- tests/test_release_workflow_event_provenance.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_release_workflow_event_provenance.py b/tests/test_release_workflow_event_provenance.py index cf29a7f7b..440b505c8 100644 --- a/tests/test_release_workflow_event_provenance.py +++ b/tests/test_release_workflow_event_provenance.py @@ -31,7 +31,16 @@ def test_required_generic_validation_workflows_checkout_recorded_head() -> None: text = (REPO / relative).read_text(encoding="utf-8") checkout_count = text.count("uses: actions/checkout@") assert checkout_count >= 1, relative - assert text.count(exact_ref) == checkout_count, relative + if relative == ".github/workflows/ci.yml": + # Source-validation and release-gate jobs still checkout the + # PR's actual head. The two Action dogfood jobs intentionally + # exercise GitHub's evaluated synthetic merge-preview commit, + # whose SHA is github.sha for a pull_request event. + merge_ref = "ref: ${{ github.sha }}" + assert text.count(merge_ref) == 2, relative + assert text.count(exact_ref) == checkout_count - 2, relative + else: + assert text.count(exact_ref) == checkout_count, relative def test_release_collector_supports_dispatch_only_filter() -> None: From f241299fbc385db88afbf05de140361fac9651ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:46 -0700 Subject: [PATCH 030/104] integration(action): import bounded PR source identity contract docs/PR_SOURCE_BINDING_V1.md --- docs/PR_SOURCE_BINDING_V1.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/PR_SOURCE_BINDING_V1.md diff --git a/docs/PR_SOURCE_BINDING_V1.md b/docs/PR_SOURCE_BINDING_V1.md new file mode 100644 index 000000000..31f799b8c --- /dev/null +++ b/docs/PR_SOURCE_BINDING_V1.md @@ -0,0 +1,34 @@ +# PR source identity contract — merge-preview development v1 + +**Security and release prerequisite; non-controlling.** This change addresses [#222](https://github.com/fraware/open-verification-kernel/issues/222). It makes the Action's *default PR diff* refer to the same Git commit as the existing `--head-sha` evidence subject. It is **not** a complete authenticated verification receipt or a proof of source-to-model fidelity. + +## Distinct objects + +For a GitHub `pull_request`, the event reports `base.sha` and `head.sha`. The Actions `github.sha` normally identifies a GitHub-generated merge-preview commit, not the contributor head. `actions/checkout` normally checks out that merge preview. The previous Action built `git diff base head` yet wrote `github.sha` as the claim subject and targeted GitHub checks at `github.sha`. These differ when the merge tree differs from the PR head tree. A reliable system must never silently treat them as one object. + +## Bounded new behavior + +`scripts/prepare_verified_change_input.py`, invoked from `action.yml` after checkout, now: + +1. Requires the actual checked-out commit to equal the full `github.sha` provided by the GitHub event. In the ordinary PR path, reads the commit's raw parent headers and requires precisely two parents: event `base.sha` and `pull_request.head.sha`, in that order. +2. Requires the base commit object to be available (attempting an explicit shallow fetch otherwise), with **no fallback** to `HEAD^ HEAD` or a different comparison on fetch failure. Computes `git diff --no-ext-diff --binary event-base checkout-merge` and its SHA-256. For non-PR push, compares the checked-out first parent to the checked-out commit. A root commit uses `git show`. +3. Writes a versioned `ovk-source-binding.json` sidecar recording the two event identities, evaluated commit/tree identity, exact diff SHA-256, input comparison mode, explicit limits on authentication/relying policy and whether input was structurally checked against the checkout. +4. Rejects mismatching or malformed commit IDs, incompatible PR parentage, and `pull_request_target`. Caller-selected change input remains supported **in advisory mode only**, and the sidecar marks it `caller_supplied_unbound`. A caller-selected input is refused in strict mode. To prevent an unbound advisory fixture from becoming a misleading shareable check or comment, the Action suppresses automatic GitHub Check and PR-comment emission for caller-selected input, and emits an explicit warning. The local advisory evidence can still be inspected as an experiment. +5. Makes this sidecar available among the existing Action artifacts. Existing `ovk check` evidence and check API calls still use `github.sha`, now consistently describing the checked-out **merge preview** for the ordinary default PR input. + +The new test creates real base/head/merge Git commits, asserts the chosen binary diff, introduces wrong base/head/checkout identifiers, tests a contributor-only checkout, opaque inputs, `pull_request_target`, push, and failure without writing a misleading receipt. + +## Important limitations + +- **Not source authenticity proof.** The GitHub event metadata, runner, checkout and storage remain unverified external observations. Raw parent shape is not a signed source/target validity certificate. +- **Not a signed evidence link.** This sidecar is **not yet incorporated into the canonical signed `EvidenceBundle` or `ovk-attestation-envelope`**. Its flags `bound_into_signed_evidence_envelope=false` and `production_reliance_authorized=false` are literal. Do not publish it as a portable signed verified-PR receipt. +- **Not general source correctness.** Diff inspection, compiled Assurance IR coverage and checker trust are separate. A structurally bound diff establishes *which Git tree was compared*, not that a machine claim comprehensively represents application source behavior. +- **Not PR-head verification.** The admitted default compares the event **base** to the synthetic **merge preview**. GitHub Checks created using `github.sha` are attributed to that merge SHA. For reviewer-facing PR-head checks, independently model *presentation SHA* and *verified subject SHA* before changing the Checks API target. +- **Shallow and private checkout prerequisites.** The default one-commit checkout may lack the base object. An explicit `fetch-depth: 2` on `actions/checkout` is recommended for a normal merge-preview PR; on some private/fork checkouts a missing base will require authenticated fetching. If absent and unfetchable, the Action fails closed. The first-user templates must be updated to guarantee this on the actual published Action revision. +- **Existing strict-profile no-go persists.** Historical issuer authorization, source-profile calibration, revocation, exact release identity and trustworthy evidence transport remain independently open under #25, #178, #195 and #205. No new `VALID`, production reuse, or automatic merge authorization. + +## Adversarial qualification + +Run `pytest -q tests/test_action_pr_source_binding_v1.py` and full repository CI on the exact PR SHA. The normal `action_default_diff_dogfood` workflow must pass from a genuine GitHub pull request event, exercising the new strict parent binding. A test pass on an earlier SHA does not authorize a later commit. + +The next qualified iteration will integrate this sidecar's exact comparison/identity into the signed evidence envelope with a typed provenance contract and distinguish source-subject identity from GitHub Check presentation identity, including base drift, PR updates, and fork permissions. That later work requires independent oracle evaluation and reviewer usefulness. This PR is a bounded correctness prerequisite for the same open-source CLI/Action product, **not** a new speculative subsystem. From a9ca3a42101d5d26b06269acd81704e39c134cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:49 -0700 Subject: [PATCH 031/104] integration(action): import bounded PR source identity contract scripts/prepare_verified_change_input.py --- scripts/prepare_verified_change_input.py | 183 +++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 scripts/prepare_verified_change_input.py diff --git a/scripts/prepare_verified_change_input.py b/scripts/prepare_verified_change_input.py new file mode 100644 index 000000000..4cb133acb --- /dev/null +++ b/scripts/prepare_verified_change_input.py @@ -0,0 +1,183 @@ +"""Construct source-bound Action diff with a separate non-authoritative receipt. + +For pull_request events the default checkout is a synthetic merge preview. +An observed base-to-merge diff must match the actual checked-out source. +Caller-provided inputs are explicitly unbound. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path + +SHA = re.compile(r"^[0-9a-f]{40}(?:[0-9a-f]{24})?$") + + +class BindingError(ValueError): + pass + + +def git(*args: str) -> bytes: + result = subprocess.run( + ["git", *args], capture_output=True, check=False, + ) + if result.returncode != 0: + raise BindingError( + f"git {args[0]} failed ({result.returncode}): " + f"{result.stderr.decode('utf-8', errors='replace')[:300]}" + ) + return result.stdout + + +def sha(value: str, required: bool = False) -> str | None: + if not value and not required: + return None + if not SHA.fullmatch(value): + raise BindingError("missing or malformed full commit SHA") + return value + + +def parents() -> list[str]: + # Raw commit headers retain parents even on shallow GitHub checkouts. + header = git("cat-file", "-p", "HEAD").split(b"\n\n", 1)[0] + result = [ + line.split(b" ", 1)[1].decode("ascii") + for line in header.splitlines() if line.startswith(b"parent ") + ] + if not all(SHA.fullmatch(item) for item in result): + raise BindingError("malformed parent in checked-out commit") + return result + + +def base_available(base: str) -> None: + test = subprocess.run( + ["git", "cat-file", "-e", f"{base}^{{commit}}"], + capture_output=True, check=False, + ) + if test.returncode != 0: + git("fetch", "--no-tags", "--depth=1", "origin", base) + git("cat-file", "-e", f"{base}^{{commit}}") + + +def prepare( + *, event_name: str, github_sha: str, base_sha: str = "", + pr_head_sha: str = "", changed_files: str = "", mode: str = "advisory", +) -> tuple[str, bytes, dict]: + checkout = git("rev-parse", "HEAD").decode().strip() + tree = git("rev-parse", "HEAD^{tree}").decode().strip() + if not SHA.fullmatch(checkout) or not SHA.fullmatch(tree): + raise BindingError("invalid checked-out Git object") + if checkout != sha(github_sha, required=True): + raise BindingError("checked-out commit differs from GITHUB_SHA") + if event_name == "pull_request_target": + raise BindingError("privileged pull_request_target is not admitted") + base = None + pr_head = None + if event_name == "pull_request": + base = sha(base_sha, required=True) + pr_head = sha(pr_head_sha, required=True) + if parents() != [base, pr_head]: + raise BindingError("PR merge parents disagree with event base and head") + base_available(base) + elif base_sha or pr_head_sha: + raise BindingError("PR identity parameters supplied for non-PR event") + + if changed_files: + if mode == "strict": + raise BindingError("caller-supplied diff cannot authorize strict review") + path = Path(changed_files) + if "\n" in changed_files or "\r" in changed_files or not path.is_file(): + raise BindingError("invalid or missing changed-files input") + diff = path.read_bytes() + input_path = str(path) + comparison = "caller_supplied_unbound" + bound = False + elif event_name == "pull_request": + diff = git("diff", "--no-ext-diff", "--binary", base, checkout) + input_path = ".ovk-pr.diff" + comparison = "event_base_to_verified_merge_preview" + bound = True + elif parents(): + diff = git("diff", "--no-ext-diff", "--binary", "HEAD^", "HEAD") + input_path = ".ovk-pr.diff" + comparison = "checked_out_first_parent_to_head" + bound = True + else: + diff = git("show", "--format=", "--no-ext-diff", "--binary", "HEAD") + input_path = ".ovk-pr.diff" + comparison = "checked_out_root_commit" + bound = True + + receipt = { + "schema_version": "ovk.pr_source_input_binding.v1-development", + "event_name": event_name, + "event_base_sha": base, + "event_pr_head_sha": pr_head, + "evaluated_checkout_commit_sha": checkout, + "evaluated_checkout_tree_sha": tree, + "evidence_subject_sha": checkout, + "input_comparison": comparison, + "input_diff_sha256": hashlib.sha256(diff).hexdigest(), + "input_structurally_bound_to_checkout": bound, + "independent_event_authentication": False, + "bound_into_signed_evidence_envelope": False, + "production_reliance_authorized": False, + } + return input_path, diff, receipt + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--event-name", required=True) + parser.add_argument("--github-sha", required=True) + parser.add_argument("--base-sha", default="") + parser.add_argument("--pr-head-sha", default="") + parser.add_argument("--changed-files", default="") + parser.add_argument("--mode", default="advisory", choices=("advisory", "strict")) + parser.add_argument("--input-path-output", type=Path, default=Path(".ovk-effective-input-path")) + parser.add_argument("--receipt-output", type=Path, default=Path("ovk-source-binding.json")) + args = parser.parse_args(argv) + try: + # These files are generated observations, never reusable input evidence. + # Clear earlier invocations before validating the current source identity. + output_paths = (args.input_path_output, args.receipt_output) + resolved_outputs = tuple(output.resolve() for output in output_paths) + output_alias = resolved_outputs[0] == resolved_outputs[1] + input_alias = bool(args.changed_files) and Path(args.changed_files).resolve() in resolved_outputs + for output in output_paths: + if output.is_dir() and not output.is_symlink(): + raise BindingError("generated output path is a directory") + output.unlink(missing_ok=True) + # Reject aliases only after removing the potentially stale reserved output. + if output_alias: + raise BindingError("handoff and receipt output paths must differ") + if input_alias: + raise BindingError("changed-files input overlaps generated output") + path, content, receipt = prepare( + event_name=args.event_name, github_sha=args.github_sha, + base_sha=args.base_sha, pr_head_sha=args.pr_head_sha, + changed_files=args.changed_files, mode=args.mode, + ) + except (BindingError, OSError, UnicodeError) as exc: + print(f"OVK source binding rejected: {exc}", file=sys.stderr) + return 2 + if path == ".ovk-pr.diff": + Path(path).write_bytes(content) + # The composite Action reads this file from Bash; a Windows CRLF suffix + # would become part of the path after command substitution. Emit exact LF. + args.input_path_output.write_bytes((path + "\n").encode("utf-8")) + args.receipt_output.write_text( + json.dumps(receipt, sort_keys=True, indent=2) + "\n", encoding="utf-8", + ) + print(f"OVK source input: {receipt['input_comparison']} [{receipt['input_diff_sha256']}]") + if not receipt["input_structurally_bound_to_checkout"]: + print("OVK warning: caller-supplied input is not bound to checked-out source") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 3c0fd5a4f88b586d7b180d4913ee7067ee70f06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:51 -0700 Subject: [PATCH 032/104] integration(action): import bounded PR source identity contract tests/test_action_pr_source_binding_v1.py --- tests/test_action_pr_source_binding_v1.py | 226 ++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 tests/test_action_pr_source_binding_v1.py diff --git a/tests/test_action_pr_source_binding_v1.py b/tests/test_action_pr_source_binding_v1.py new file mode 100644 index 000000000..1698297e0 --- /dev/null +++ b/tests/test_action_pr_source_binding_v1.py @@ -0,0 +1,226 @@ +"""Adversarial Git-object tests for source/diff binding in the Action. + +These are real Git commits and a two-parent merge-preview, not mocked SHA +strings. No network or GitHub credentials are involved. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from scripts import prepare_verified_change_input as binding + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="Git required") + + +def git(*args: str) -> str: + proc = subprocess.run( + ["git", *args], check=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + return proc.stdout.strip() + + +@pytest.fixture +def example_repo(tmp_path, monkeypatch): + repo = tmp_path / "consumer" + repo.mkdir() + monkeypatch.chdir(repo) + git("init", "-b", "main") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "OVK Test") + git("config", "core.autocrlf", "false") + (repo / "base.txt").write_text("base\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "base") + base = git("rev-parse", "HEAD") + git("checkout", "-b", "contributor") + (repo / "pr.txt").write_text("head\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "head") + pr_head = git("rev-parse", "HEAD") + git("checkout", "main") + git("merge", "--no-ff", "contributor", "-m", "merge-preview") + merge = git("rev-parse", "HEAD") + assert merge != pr_head + assert binding.parents() == [base, pr_head] + return {"repo": repo, "base": base, "pr_head": pr_head, "merge": merge} + + +def make_args(record): + return { + "event_name": "pull_request", + "base_sha": record["base"], + "pr_head_sha": record["pr_head"], + "github_sha": record["merge"], + } + + +def test_real_merge_preview_checks_the_tree_it_labels(example_repo): + path, diff, receipt = binding.prepare(**make_args(example_repo)) + expected = subprocess.run( + ["git", "diff", "--no-ext-diff", "--binary", + example_repo["base"], example_repo["merge"]], + check=True, stdout=subprocess.PIPE, + ).stdout + assert path == ".ovk-pr.diff" + assert diff == expected + assert b"pr.txt" in diff + assert receipt["event_pr_head_sha"] != receipt["evaluated_checkout_commit_sha"] + assert receipt["evidence_subject_sha"] == example_repo["merge"] + assert receipt["event_base_sha"] == example_repo["base"] + assert receipt["input_comparison"] == "event_base_to_verified_merge_preview" + assert receipt["input_diff_sha256"] == hashlib.sha256(diff).hexdigest() + assert receipt["input_structurally_bound_to_checkout"] is True + assert receipt["bound_into_signed_evidence_envelope"] is False + assert receipt["production_reliance_authorized"] is False + + +@pytest.mark.parametrize("mutation", ["head", "base", "checkout"]) +def test_identity_confusion_rejected(example_repo, mutation): + options = make_args(example_repo) + if mutation == "head": + options["pr_head_sha"] = example_repo["base"] + elif mutation == "base": + options["base_sha"] = example_repo["pr_head"] + else: + options["github_sha"] = example_repo["pr_head"] + with pytest.raises(binding.BindingError): + binding.prepare(**options) + + +def test_non_merge_pr_checkout_rejected(example_repo): + git("checkout", "contributor") + options = make_args(example_repo) + options["github_sha"] = example_repo["pr_head"] + with pytest.raises(binding.BindingError, match="merge parents"): + binding.prepare(**options) + + +@pytest.mark.parametrize("payload", [b"caller-selected input\n", b"caller-selected input\r\n"]) +def test_external_input_is_advisory_and_explicitly_unbound(example_repo, payload): + # Bytes must be stable across OSes: Windows text-mode writes translate LF to CRLF. + custom = example_repo["repo"] / "manual.diff" + custom.write_bytes(payload) + path, diff, receipt = binding.prepare( + **make_args(example_repo), changed_files=str(custom), + ) + assert path == str(custom) + assert diff == payload + assert receipt["input_diff_sha256"] == hashlib.sha256(payload).hexdigest() + assert receipt["input_comparison"] == "caller_supplied_unbound" + assert receipt["input_structurally_bound_to_checkout"] is False + with pytest.raises(binding.BindingError, match="cannot authorize strict"): + binding.prepare(**make_args(example_repo), changed_files=str(custom), mode="strict") + + +def test_caller_input_does_not_hide_mismatched_checkout(example_repo): + custom = example_repo["repo"] / "manual.diff" + custom.write_text("caller-selected input", encoding="utf-8") + options = make_args(example_repo) + options["github_sha"] = example_repo["base"] + with pytest.raises(binding.BindingError): + binding.prepare(**options, changed_files=str(custom)) + + +def test_privileged_event_refused(example_repo): + with pytest.raises(binding.BindingError, match="pull_request_target"): + binding.prepare( + event_name="pull_request_target", github_sha=example_repo["merge"], + ) + + +def test_bad_sha_and_non_pr_pr_metadata_fail_closed(example_repo): + with pytest.raises(binding.BindingError): + binding.prepare(event_name="pull_request", github_sha="HEAD") + with pytest.raises(binding.BindingError, match="non-PR"): + binding.prepare( + event_name="push", github_sha=example_repo["merge"], + pr_head_sha=example_repo["pr_head"], + ) + + +def test_cli_generates_separate_observational_receipt(example_repo): + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["merge"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + ]) == 0 + path = Path(".ovk-effective-input-path").read_text(encoding="utf-8").strip() + receipt = json.loads(Path("ovk-source-binding.json").read_text(encoding="utf-8")) + assert path == ".ovk-pr.diff" + # Bash command substitution strips LF, not CR. This must be exact on Windows. + assert Path(".ovk-effective-input-path").read_bytes() == b".ovk-pr.diff\n" + assert receipt["input_diff_sha256"] == hashlib.sha256( + Path(path).read_bytes() + ).hexdigest() + assert receipt["bound_into_signed_evidence_envelope"] is False + + +def test_source_binding_failure_emits_no_misleading_new_receipt(example_repo): + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["base"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + ]) == 2 + assert not Path("ovk-source-binding.json").exists() + + +def test_rejected_source_binding_discards_previous_generated_outputs(example_repo): + # A failed PR identity check must not leave old handoff/receipt files that + # the always-running artifact upload step could mistake for current output. + Path(".ovk-effective-input-path").write_bytes(b"stale.diff\\n") + Path("ovk-source-binding.json").write_bytes(b'{"forged": true}\\n') + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["base"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + ]) == 2 + assert not Path(".ovk-effective-input-path").exists() + assert not Path("ovk-source-binding.json").exists() + + +def test_caller_input_cannot_alias_generated_receipt(example_repo): + content = b"caller supplied input" + Path("ovk-source-binding.json").write_bytes(content) + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["merge"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + "--changed-files", "ovk-source-binding.json", + ]) == 2 + # Reserved generated artifacts must be removed even when used as inputs. + assert not Path("ovk-source-binding.json").exists() + + +def test_identical_generated_output_paths_fail_closed(example_repo): + reserved = Path("shared-receipt.json") + reserved.write_bytes(b"old generated data") + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["merge"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + "--input-path-output", str(reserved), + "--receipt-output", str(reserved), + ]) == 2 + assert not reserved.exists() + + +def test_non_pr_push_uses_actual_checkout_parent(example_repo): + _, diff, receipt = binding.prepare( + event_name="push", github_sha=example_repo["merge"], + ) + assert receipt["input_comparison"] == "checked_out_first_parent_to_head" + assert receipt["input_structurally_bound_to_checkout"] is True + assert b"pr.txt" in diff From 3a69c5ce75d20f8ffae05277f5b6a791c1cecfcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:25:53 -0700 Subject: [PATCH 033/104] integration(action): import bounded PR source identity contract tests/test_action_source_binding_workflow_contract.py --- ...action_source_binding_workflow_contract.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/test_action_source_binding_workflow_contract.py diff --git a/tests/test_action_source_binding_workflow_contract.py b/tests/test_action_source_binding_workflow_contract.py new file mode 100644 index 000000000..5243342ec --- /dev/null +++ b/tests/test_action_source_binding_workflow_contract.py @@ -0,0 +1,51 @@ +"""Assert the composite Action and CI invoke the bounded source identity contract.""" +from pathlib import Path +import yaml +ROOT = Path(__file__).resolve().parents[1] + +def test_action_executes_identity_helper_and_uploads_receipt(): + action = yaml.safe_load((ROOT / "action.yml").read_text()) + steps = action["runs"]["steps"] + prepare = [s for s in steps if s.get("name") == "Prepare change input"] + assert len(prepare) == 1 + script = prepare[0]["run"] + assert "scripts/prepare_verified_change_input.py" in script + assert "set -euo pipefail" in script + assert "--github-sha" in script and "--pr-head-sha" in script + assert "--base-sha" in script and "--mode" in script + assert "|| true" not in script + upload = next(s for s in steps if s.get("name") == "Upload OVK artifacts") + assert "ovk-source-binding.json" in upload["with"]["path"] + +def test_dogfood_uses_merge_preview_and_rejects_unbound_strict(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + jobs = workflow["jobs"] + for name in ("action_default_diff_dogfood", "action_strict_dogfood"): + steps = jobs[name]["steps"] + checkout = next(s for s in steps if "actions/checkout@" in s.get("uses", "")) + assert checkout["with"]["ref"] == "${{ github.sha }}" + assert checkout["with"]["fetch-depth"] == 2 + assert checkout["with"]["persist-credentials"] is False + strict = jobs["action_strict_dogfood"] + assert strict["permissions"] == {"contents": "read"} + candidate = next(s for s in strict["steps"] if s.get("name", "").startswith("Strict Action")) + assert candidate["with"]["mode"] == "strict" + assert candidate["with"]["changed-files"] + assert candidate["with"]["emit-check"] == "false" + assert "test ! -f ovk-evidence.json" in strict["steps"][-1]["run"] + +def test_docs_reject_unproved_signed_source_binding(): + docs = (ROOT / "docs/PR_SOURCE_BINDING_V1.md").read_text() + assert "not yet incorporated" in docs + assert "production_reliance_authorized=false" in docs + +def test_unbound_caller_diff_never_emits_a_shareable_action_check_or_comment(): + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] + check = next(s for s in steps if s.get("name") == "Emit OVK GitHub check (check path)") + comment = next(s for s in steps if s.get("name") == "Post OVK pull-request comment") + warning = next(s for s in steps if s.get("name") == "Disclose unbound caller-selected inputs") + assert "inputs.changed-files == ''" in check["if"] + assert "inputs.changed-files == ''" in comment["if"] + assert "caller-supplied" in warning["run"] + assert "inputs.changed-files != ''" in warning["if"] From e690b02e68b9866eed55b36bcf01c8ef59de958f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:29:03 -0700 Subject: [PATCH 034/104] test(action): qualify ordering of fail-closed installation and PR source binding --- .../test_action_combined_identity_contract.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/test_action_combined_identity_contract.py diff --git a/tests/test_action_combined_identity_contract.py b/tests/test_action_combined_identity_contract.py new file mode 100644 index 000000000..07a0b198c --- /dev/null +++ b/tests/test_action_combined_identity_contract.py @@ -0,0 +1,70 @@ +"""Regression contract for combined Action version and source identity. + +These assertions cover the ordering and fail-closed composition of the +independently tested installer and source-binding helper. They do not establish +GitHub event authenticity, source-to-model fidelity, or relying authority. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def _steps() -> list[dict]: + """Load exactly the composite steps delivered to downstream Action users.""" + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + assert action["runs"]["using"] == "composite" + return action["runs"]["steps"] + + +def test_version_identity_precedes_verified_source_binding() -> None: + """Failure to resolve an explicitly requested version must stop the Action.""" + steps = _steps() + names = [step.get("name") for step in steps] + for required in ("Install OVK package", "Initialize verification directory if needed", + "Prepare change input", "Run OVK check path"): + assert names.count(required) == 1 + + install_i = names.index("Install OVK package") + init_i = names.index("Initialize verification directory if needed") + prepare_i = names.index("Prepare change input") + check_i = names.index("Run OVK check path") + assert install_i < init_i < prepare_i < check_i + + install = steps[install_i] + assert install["shell"] == "bash" + assert "set -euo pipefail" in install["run"] + assert 'pip install "open-verification-kernel==${OVK_PACKAGE_VERSION}"' in install["run"] + assert '|| install_from_checkout' not in install["run"] + assert "install_from_checkout" in install["run"] + + prepare = steps[prepare_i] + assert "set -euo pipefail" in prepare["run"] + assert "scripts/prepare_verified_change_input.py" in prepare["run"] + assert prepare["env"]["OVK_GITHUB_SHA"] == "${{ github.sha }}" + assert prepare["env"]["OVK_EVENT_HEAD_SHA"] == "${{ github.event.pull_request.head.sha }}" + assert prepare["env"]["OVK_EVENT_BASE_SHA"] == "${{ github.event.pull_request.base.sha }}" + assert "--github-sha" in prepare["run"] and "--base-sha" in prepare["run"] + assert "--pr-head-sha" in prepare["run"] + + +def test_unbound_manual_diff_cannot_be_published_by_default_check_path() -> None: + """An advisory caller-selected diff is explicitly outside the bound PR verdict.""" + steps = _steps() + by_name = {step["name"]: step for step in steps} + check = by_name["Emit OVK GitHub check (check path)"] + comment = by_name["Post OVK pull-request comment"] + assert "inputs.changed-files == ''" in check["if"] + assert "inputs.changed-files == ''" in comment["if"] + assert "inputs.verification-manifest != ''" in comment["if"] + + upload = by_name["Upload OVK artifacts"] + assert "ovk-source-binding.json" in upload["with"]["path"] + # A receipt sidecar exists, but it remains outside the signed envelope. + source_doc = (ROOT / "docs/PR_SOURCE_BINDING_V1.md").read_text(encoding="utf-8") + assert "bound_into_signed_evidence_envelope=false" in source_doc From 89c23ef8a841fd32966d7263f414441a0e0139c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:41:21 -0700 Subject: [PATCH 035/104] integration(pilot): preserve #217 evidence-adjudication boundary in scripts/ingest_external_pilot_metrics.py --- scripts/ingest_external_pilot_metrics.py | 54 ++++++++++-------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/scripts/ingest_external_pilot_metrics.py b/scripts/ingest_external_pilot_metrics.py index bfb90662a..9ba5c3b44 100644 --- a/scripts/ingest_external_pilot_metrics.py +++ b/scripts/ingest_external_pilot_metrics.py @@ -40,14 +40,6 @@ def _iso_timestamp() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") -def _find_file(artifacts_dir: Path, name: str) -> Path | None: - direct = artifacts_dir / name - if direct.is_file(): - return direct - matches = sorted(artifacts_dir.rglob(name)) - return matches[0] if matches else None - - def _median_elapsed_ms(results: list[dict[str, Any]]) -> float | None: timings = [float(item["elapsed_ms"]) for item in results if "elapsed_ms" in item] if not timings: @@ -55,16 +47,6 @@ def _median_elapsed_ms(results: list[dict[str, Any]]) -> float | None: return float(statistics.median(timings)) -def _false_positive_rate(results: list[dict[str, Any]]) -> float | None: - evaluated = [item for item in results if "passed" in item] - if not evaluated: - return None - false_positives = sum( - 1 for item in evaluated if not item.get("passed") and item.get("merge_recommendation") == "block" - ) - return false_positives / len(evaluated) - - def _blocked_count(results: list[dict[str, Any]]) -> int | None: evaluated = [item for item in results if "merge_recommendation" in item] if not evaluated: @@ -95,9 +77,10 @@ def _metrics_from_artifacts(artifacts_dir: Path) -> dict[str, Any]: results = list(pilot_report.get("results", [])) metrics["prs_evaluated"] = len(results) or None metrics["prs_blocked"] = _blocked_count(results) - metrics["false_positive_rate"] = _false_positive_rate(results) - if metrics["false_positive_rate"] is not None and metrics["prs_evaluated"]: - metrics["false_positives"] = round(metrics["false_positive_rate"] * metrics["prs_evaluated"]) + # 'passed' and 'merge_recommendation' are OVK's own output labels. + # They cannot independently adjudicate a false positive or measure + # false-negative risk. Only a separately reviewed external report may + # supply false_positives / false_positive_rate. metrics["median_check_latency_ms"] = _median_elapsed_ms(results) if evidence_path and evidence_path.exists(): @@ -111,12 +94,10 @@ def _metrics_from_artifacts(artifacts_dir: Path) -> dict[str, Any]: elif isinstance(timing, (int, float)): metrics["median_check_latency_ms"] = float(timing) - external_report = _find_file(artifacts_dir, "external_pilot_report.json") - if external_report is not None: - report = read_json_file(external_report) - for report_key, row_key in REPORT_FIELD_MAP.items(): - if report_key in report and report[report_key] is not None: - metrics[row_key] = report[report_key] + # Artifact directories are not an adjudication authority. Even if they + # contain an external_pilot_report.json, never auto-adopt its verdicts, + # repository identity or strict_enabled flag. A report is accepted only + # via the explicit, separately reviewed --report input. return metrics @@ -127,8 +108,10 @@ def _apply_report(row: dict[str, Any], report_path: Path) -> dict[str, Any]: for report_key, row_key in REPORT_FIELD_MAP.items(): if report_key in report and report[report_key] is not None: updated[row_key] = report[report_key] - if "repository" in report and report["repository"]: - updated["repository"] = report["repository"] + # The CLI --repo argument is authoritative for the registry row. + # Never let a separately supplied report silently redirect ingestion. + if report.get("repository") and report["repository"] != updated["repository"]: + raise ValueError("external pilot report repository mismatch") return updated @@ -159,10 +142,19 @@ def build_pilot_row( if existing is not None: row.update({key: value for key, value in existing.items() if value is not None or key in row}) row["repository"] = repo + if artifacts_dir is not None and artifacts_dir.is_dir(): + artifact_metrics = _metrics_from_artifacts(artifacts_dir) + if artifact_metrics.get("prs_evaluated") is not None: + # A new artifact cohort invalidates older adjudicated error + # counts. Avoid carrying forward a historical zero as evidence + # that the current, unreviewed cohort had no false positives. + row["false_positives"] = None + row["false_positive_rate"] = None + row.update({key: value for key, value in artifact_metrics.items() if value is not None}) if report_path is not None and report_path.exists(): + # A reviewer-provided report is the only source of error labels. + # Self-reports still need independent verification before promotion. row = _apply_report(row, report_path) - if artifacts_dir is not None and artifacts_dir.is_dir(): - row.update({key: value for key, value in _metrics_from_artifacts(artifacts_dir).items() if value is not None}) row["status"] = _infer_status(row) return row From 3658b0f322a3b1ef09c2973d819bd3a52400779d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:41:24 -0700 Subject: [PATCH 036/104] integration(pilot): preserve #217 evidence-adjudication boundary in tests/test_ingest_external_pilot_metrics.py --- tests/test_ingest_external_pilot_metrics.py | 152 +++++++++++++++++++- 1 file changed, 151 insertions(+), 1 deletion(-) diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index 0544086d0..28d562908 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -10,6 +10,7 @@ from ovk.core.json_io import read_json_file, write_json_file from ovk.core.schema_validation import require_schema_valid from scripts.ingest_external_pilot_metrics import ( + _metrics_from_artifacts, build_pilot_row, load_registry, upsert_pilot_row, @@ -39,7 +40,14 @@ def test_build_pilot_row_from_artifacts(tmp_path: Path) -> None: encoding="utf-8", ) - row = build_pilot_row("example/oss-pilot", artifacts_dir=artifacts_dir, ovk_version="1.2.0") + # The self-report must be separately selected as an adjudication source; + # merely placing it inside downloaded CI artifacts is insufficient. + row = build_pilot_row( + "example/oss-pilot", + artifacts_dir=artifacts_dir, + report_path=artifacts_dir / "external_pilot_report.json", + ovk_version="1.2.0", + ) assert row["repository"] == "example/oss-pilot" assert row["prs_evaluated"] == 4 assert row["false_positive_rate"] == 0.0 @@ -153,3 +161,145 @@ def test_validate_registry_rejects_invalid_row(tmp_path: Path) -> None: } with pytest.raises(ValueError, match="external pilots registry failed schema validation"): validate_registry(registry) + + +def test_checker_decisions_cannot_self_adjudicate_false_positives( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A blocked result is not independent evidence of a false alarm.""" + raw = tmp_path / "pilot-report.json" + raw.write_text( + json.dumps({ + "results": [ + {"passed": False, "merge_recommendation": "block", "elapsed_ms":12}, + {"passed": True, "merge_recommendation": "allow", "elapsed_ms":14}, + ] + }), + encoding="utf-8", + ) + monkeypatch.setattr( + "scripts.collect_pilot_metrics.discover_artifacts_dir", + lambda _: (raw, None, None), + ) + metrics = _metrics_from_artifacts(tmp_path) + assert metrics["prs_evaluated"] == 2 + assert metrics["prs_blocked"] == 1 + assert metrics["median_check_latency_ms"] == 13.0 + assert "false_positives" not in metrics + assert "false_positive_rate" not in metrics + + +def test_new_artifacts_clear_stale_external_error_labels( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Old reviewer labels must not be reused for a new unreviewed cohort.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: {"prs_evaluated": 7, "prs_blocked": 2}, + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row( + "example/oss-pilot", existing=existing, artifacts_dir=tmp_path + ) + assert row["prs_evaluated"] == 7 + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + + +def test_external_report_supplies_separate_reviewed_labels( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Explicit reviewer-provided labels are retained, not inferred.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: {"prs_evaluated": 4, "prs_blocked": 2}, + ) + report = tmp_path / "reviewed.json" + report.write_text( + json.dumps({ + "repository": "example/oss-pilot", + "false_positives": 1, + "false_positive_rate": 0.25, + "notes": "Separately adjudicated by an external reviewer.", + }), + encoding="utf-8", + ) + row = build_pilot_row( + "example/oss-pilot", artifacts_dir=tmp_path, report_path=report + ) + assert row["prs_evaluated"] == 4 + assert row["false_positives"] == 1 + assert row["false_positive_rate"] == 0.25 + + +def test_embedded_artifact_self_report_cannot_set_error_or_strictness( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Embedded reports are not an approved independent source of labels.""" + pilot = tmp_path / "pilot-report.json" + pilot.write_text( + json.dumps({"results": [{"passed": True, "merge_recommendation": "allow", "elapsed_ms": 12}]}), + encoding="utf-8", + ) + embedded = tmp_path / "external_pilot_report.json" + embedded.write_text( + json.dumps({ + "repository": "attacker/redirected", + "prs_evaluated": 1000, + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": True, + "notes": "unreviewed data in artifact bundle", + }), + encoding="utf-8", + ) + monkeypatch.setattr( + "scripts.collect_pilot_metrics.discover_artifacts_dir", + lambda _: (pilot, None, None), + ) + row = build_pilot_row("example/oss-pilot", artifacts_dir=tmp_path) + assert row["repository"] == "example/oss-pilot" + assert row["prs_evaluated"] == 1 + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + assert row["strict_enabled"] is False + assert row["status"] == "advisory" + + +def test_explicit_report_must_not_redirect_registry_identity(tmp_path: Path) -> None: + """Selected reporter may provide observations, not reassign the subject.""" + report = tmp_path / "report.json" + report.write_text( + json.dumps({ + "repository": "other/tenant", + "false_positives": 0, + "false_positive_rate": 0.0, + }), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="repository mismatch"): + build_pilot_row("example/oss-pilot", report_path=report) + + +def test_explicit_report_with_matching_repository_still_supported(tmp_path: Path) -> None: + report = tmp_path / "report.json" + report.write_text( + json.dumps({ + "repository": "example/oss-pilot", + "false_positives": 1, + "false_positive_rate": 0.25, + }), + encoding="utf-8", + ) + row = build_pilot_row("example/oss-pilot", report_path=report) + assert row["repository"] == "example/oss-pilot" + assert row["false_positives"] == 1 + assert row["false_positive_rate"] == 0.25 From 4c36acfd9cd562c30cf4a85a5e5a38e7b07f0549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:41:26 -0700 Subject: [PATCH 037/104] integration(pilot): preserve #217 evidence-adjudication boundary in docs/templates/external_pilot_report.template.json --- .../external_pilot_report.template.json | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/templates/external_pilot_report.template.json b/docs/templates/external_pilot_report.template.json index 4ef983679..3e741171b 100644 --- a/docs/templates/external_pilot_report.template.json +++ b/docs/templates/external_pilot_report.template.json @@ -2,17 +2,19 @@ "schema_version": "ovk.external_pilot_report.v1", "repository": "org/example-repo", "repository_url": "https://github.com/org/example-repo", - "check_types": ["ci_secrets"], - "advisory_start": "2026-01-01", - "advisory_end": "2026-01-15", - "prs_evaluated": 0, - "prs_blocked": 0, - "false_positives": 0, - "false_positive_rate": 0.0, + "check_types": [ + "ci_secrets" + ], + "advisory_start": null, + "advisory_end": null, + "prs_evaluated": null, + "prs_blocked": null, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": null, "strict_enabled": false, - "ovk_version_pin": "1.2.0", + "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-pilot.yml", "evidence_url": null, - "notes": "Self-reported metrics from the external pilot maintainer. Ingest with scripts/ingest_external_pilot_metrics.py." + "notes": "Template only. Leave null until measured. False-positive counts/rates require separate reviewer adjudication with explicit denominator and evidence location; tool PASS/BLOCK is not ground truth." } From 9ee333ad7c6be0281fdb52127190b0e7635c29f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:41:29 -0700 Subject: [PATCH 038/104] integration(pilot): preserve #217 evidence-adjudication boundary in docs/EXTERNAL_PILOT_PLAYBOOK.md --- docs/EXTERNAL_PILOT_PLAYBOOK.md | 157 ++++++++++++++------------------ 1 file changed, 69 insertions(+), 88 deletions(-) diff --git a/docs/EXTERNAL_PILOT_PLAYBOOK.md b/docs/EXTERNAL_PILOT_PLAYBOOK.md index 7c1942602..69b45b3ef 100644 --- a/docs/EXTERNAL_PILOT_PLAYBOOK.md +++ b/docs/EXTERNAL_PILOT_PLAYBOOK.md @@ -1,44 +1,48 @@ -# External OSS Pilot Playbook +# External OSS pilot playbook — advisory evaluation -Guide for adding OVK to a **consenting** external open-source repository. Start in advisory mode, retain both supported and unsupported PRs, measure reviewer utility and coverage, and qualify strict enforcement only through a separate safety decision. +This playbook governs **consented, non-blocking** use of Open Verification Kernel (OVK) in an independently operated repository. An external pilot is a test of customer utility and the declared verification profile, **not** proof of general source-level security, production readiness, or verified economic savings. -## Start with one check type +The current public status identifies source profiles as `executable_advisory`. No `externally_calibrated_strict` source-profile qualification is established. A small pilot must never automatically promote a check to strict or to a required merge gate. -| Check type | Best for | Starter manifest | -|------|----------|------------------| -| CI secrets | Repos with agent-authored workflow PRs | [pilot_manifest_ci_secrets.template.json](templates/pilot_manifest_ci_secrets.template.json) | -| Self-protection | Repos adding agent CI gates | `examples/pilot_repos/self_protection_only.json` | +## 0. Obtain consent and qualify the repository -Enable additional check types one at a time after each is stable on your diffs. +Before adding any GitHub workflow, obtain approval from the maintaining organization, identify a reviewer and a technically responsible operator, and agree on which evidence is safe to retain or publish. Do not install CI into someone else's repository without permission. -**Verified release distinction (2026-10-15):** the examples below pin the exact source commit of the published `v1.2.1` release; the Git tag is unsigned and its GitHub Release is not marked immutable. `OVK_PACKAGE_VERSION: 1.2.1` selects the corresponding older PyPI distribution. New guarantee-continuity reviewer work from draft #218 is **not included**. Read [current release status](CURRENT_RELEASE_STATUS.md) before installing. +Record the target repository and exact PR revision, the declared check family, its supported source constructs and exclusions, the intended high-risk operation, the reviewer decision, and the organization's actual need. A changed authentication-related file is **not** sufficient proof that OVK supports the code's authorization semantics. Keep unsupported shapes **unknown** and subject to human review. -## Step 1 — Advisory mode (no merge blocking) +Start with one available check family. The historical example manifests include: -1. Copy `docs/templates/pilot_manifest_ci_secrets.template.json` to `.verification/ci_secrets_pilot.json` in the target repo. -2. Add check input JSON under `.verification/ci_secrets/` (see `examples/ci_secrets/input_secrets_safe.json` for shape). -3. Add OVK to CI in advisory mode. Build the PR diff locally — OVK reads files, not URLs: +| Check family | Potential starting problem | Starter manifest | +| --- | --- | --- | +| CI secrets | Changes to workflow secret exposure | [CI secrets template](templates/pilot_manifest_ci_secrets.template.json) | +| Self-protection | Changes to agent CI gates | `examples/pilot_repos/self_protection_only.json` | + +The FastAPI authorization / Protected Effect path is a separate bounded advisory research profile. It must first pass source-eligibility and coverage screening; do not substitute the CI-secrets example for proof of route authorization. + +## 1. Install a released, attributable advisory build + +Use the [current release status](CURRENT_RELEASE_STATUS.md), [release procedure](RELEASE.md) and an exact vetted release/ref identity. The historic `v1.2.1` example below uses its **full 40-hex source commit**, not the unprotected tag: confirm the remote tag and distribution integrity for the installation, and never treat a draft `v1.3.0-rc.1` as a published release on the strength of green development CI. + +Copy `docs/templates/pilot_manifest_ci_secrets.template.json` into the consenting repository as `.verification/ci_secrets_pilot.json`, and replace sample inputs with real, explicitly in-scope paths. Do **not** treat `examples/ci_secrets/input_secrets_safe.json` as coverage of the adopter's code. + +The action should use least privileges and no automatic comments on untrusted fork PRs. Example starting point (adapt to the target repository's branch and trust settings): ```yaml -name: OVK Pilot +name: OVK advisory pilot on: pull_request: branches: [main] - -env: - OVK_PACKAGE_VERSION: "1.2.1" - +permissions: + contents: read jobs: ovk-advisory: runs-on: ubuntu-latest - permissions: - contents: read - # No token write permissions for fork-compatible advisory evaluation. steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 - - name: Build PR diff for OVK + persist-credentials: false + - name: Build PR diff run: | git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff @@ -49,96 +53,73 @@ jobs: use-check: "true" changed-files: ovk-pr.diff post-comment: "false" - - name: OVK advisory verify (CI secrets check) + - name: OVK advisory verification uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json bundle-output-dir: ovk-pilot-bundle post-comment: "false" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: ovk-pilot-artifacts + path: | + ovk-evidence.json + ovk-pilot-bundle/** + retention-days: 30 + if-no-files-found: warn ``` -4. Run for **two weeks** on the organization's natural PR stream, including relevant non-agent changes. Record all eligible and unsupported cases without blocking merges; if the sample is too small, extend the observation window and report attrition. - -## Step 2 — Measure - -| Measurement | Required reporting | -|--------|--------------------| -| Source-profile coverage | Eligible and unsupported PRs with exact denominators and known blind spots | -| Reviewer judgments | All findings reviewed, disagreements, time spent, independently discovered misses and unknown outcomes | -| Operational cost | Setup effort, per-PR runtime, reviewer minutes, repeat-check and repair work | -| Known-bad controls | All relevant admitted adversarial cases with versioned evidence; never a substitute for real-PR calibration | - -**A small pilot demonstrates feasibility, not production accuracy.** Report counts and denominators, rather than treating a threshold such as a 5% false-positive rate from a handful of PRs as sufficient for strict enforcement. - -Download the workflow artifact (`ovk-pilot-artifacts`) and ingest metrics into the OVK registry: - -```bash -python scripts/ingest_external_pilot_metrics.py --repo org/repo --artifacts-dir ./artifact -python scripts/render_pilot_metrics.py --registry docs/benchmarks/external-pilots-registry.json -``` - -Optional self-report JSON: copy [external_pilot_report.template.json](templates/external_pilot_report.template.json), fill measured values, and pass `--report external_pilot_report.json` to the ingest script. +Review the actual code and external actions before enabling this workflow. The full source-commit SHA pins the top-level Action, yet the historical Action may reference transitive dependencies by mutable tags; independently review that supply chain and keep the pilot non-blocking. The check stays advisory, cannot authorize a merge, and does not receive secrets from untrusted PRs. -Publish results in [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md) under **External pilots**. +## 2. Observe all naturally occurring PRs -Registry rows live in [external-pilots-registry.json](benchmarks/external-pilots-registry.json). Existing maintained-consumer reports are not independent customer evaluations. +Begin with a 14-day window. Collect **all** natural PRs in the consented scope, not a favorable selection. A target of five PRs is useful to test deployment feasibility, **not** a safety or false-negative-rate threshold. Extend the observation window if activity is low. -## Step 3 — Independent strict-enforcement qualification (not an automatic upgrade) +For each PR, record the exact base/head SHAs, source and verifier version, declared claim, supported/unsupported/unknown input, raw evidence and assumptions, report viewed by the reviewer, actual reviewer decision, adjudicated incorrect alerts, independently discovered misses, setup and review time, and fixes or rechecks. Record missing observations as missing—not as zeros. Keep private source and evidence out of the public registry unless explicitly approved. -**Do not switch a pilot to strict mode from a low measured false-positive rate alone.** Current generated source profiles are `executable_advisory`, not `externally_calibrated_strict`. +If the pipeline reports a `PASS`, communicate the **typed checker/IR claim under its exact coverage assumptions**. Never summarize it as proof that an arbitrary source application is secure. If extraction is partial or an operation is outside the admitted profile, the outcome is unresolved. -Strict enforcement must be separately authorized by the adopting organization for a *specific exact product version, supported source profile and claim*. Before using any result as a required check, document: +## 3. Compute metrics with correct provenance -1. Trusted acquisition of the actual repository inputs, source revision, policy and toolchain identities. -2. Source-to-model fidelity for the admitted constructs, together with explicit unsupported, unknown, timeout and error handling. -3. Human-adjudicated normal PRs and adversarial change tests, with denominators, missed issues, false alarms, and uncertainty reported. No arbitrary small-N accuracy threshold. -4. CI token permissions, adversarial fork PR isolation, bypass and workflow-self-modification resistance, protected branch rules, and a rollback owner. -5. Reviewer approval and monitoring for the exact admitted claim family. Unknown and incomplete outcomes do not count as positive verification. +The ingest command is: -Until those gates are independently met, remain in **advisory mode with no merge-blocking authority**. Green fixture tests do not override this constraint. +```bash +python scripts/ingest_external_pilot_metrics.py \ + --repo org/repo --artifacts-dir ./artifact \ + --report ./external_pilot_report.json +python scripts/render_pilot_metrics.py \ + --registry docs/benchmarks/external-pilots-registry.json +``` -## Step 4 — Add more check types +The optional report starts from [external_pilot_report.template.json](templates/external_pilot_report.template.json); replace its placeholders. Even if a file named `external_pilot_report.json` appears within a downloaded CI artifact directory, the ingester **does not automatically adopt its adjudication fields**. A human operator must deliberately select a separately reviewed report via `--report`, with the same repository identifier as `--repo`. Such reports remain self-reported until independently audited. `prs_evaluated`, `prs_blocked`, and latency can come from workflow artifacts. **False positives require separate human or independently specified ground-truth adjudication.** OVK's own `passed`, `merge_recommendation`, and checker `PASS/FAIL` fields are not ground-truth labels. The metric ingester must not manufacture a false-positive rate from them. -After CI secrets is stable, add check types one at a time using manifests from `examples/pilot_repos/`. Re-run advisory for each new check before strict enforcement. +Only supply `false_positives` and `false_positive_rate` if a reviewer actually classified cases and the denominator and decisions are documented in the report's linked evidence or notes. Otherwise leave both `null`. If a new artifact cohort arrives without fresh adjudication, previously reported rates are stale and must not be carried forward as current measurements. A reviewer-provided number is still **self-reported** until independently audited. -## Copy-paste workflow +Explicitly distinguish: +- **Check/recommendation rate:** what OVK returned on actual PRs. +- **Adjudicated false alarms:** the reviewer judged the tool's finding incorrect, with a defined denominator. +- **Independently found misses:** issues found by a separate source of truth that OVK did not flag, including unsupported cases. +- **Coverage/unknown fraction:** what proportion of natural PRs and relevant constructs fell outside the supported profile. +- **Economics:** reviewer minutes, CI latency/cost, integration effort and rechecks; compare with a credible baseline. -See `examples/github_workflows/pilot_fork_adopter.yml` and the full example in Step 1 above. Upload artifacts for your pilot report: +Small selected samples, even with zero observed false positives, cannot justify a general low-risk claim. -```yaml - - uses: actions/upload-artifact@v4 - with: - name: ovk-pilot-artifacts - path: | - ovk-evidence.json - ovk-pilot-bundle/** - retention-days: 30 -``` +## 4. Decision gate: continue, revise, or stop -## Reference in this repo +**Continue advisory:** the maintainer actively uses the findings and they remain appropriately scoped, with transparent unknowns and manageable review costs. -Weekly in-repo pilot workflow: `.github/workflows/pilot-dogfood.yml` with `scripts/collect_pilot_metrics.py`. +**Revise:** coverage misses important high-risk changes, the result is difficult to interpret, or false alarms dominate. Improve the *supported profile* or workflow only with evidence from actual users. -Published advisory reports (maintained consumers + infra profile): [pilots/README.md](pilots/README.md). +**Stop or select a different wedge:** the relevant guarantee is outside the verifier's modeled semantics, no reviewer values the output, or no willing maintainer participates. -## Support artifacts +**Strict enforcement is a distinct qualification project.** It needs a justified source-model correspondence, trustworthy evidence/provenance, independent profile-appropriate safety evaluation with explicit unknown handling, security review, a controlled rollback plan, repository-specific risk approval, and release/ref integrity. No numeric false-positive threshold or five-PR fixture test automatically grants strict maturity. -- Repair loop walkthrough: [AGENT_REPAIR_LOOP.md](AGENT_REPAIR_LOOP.md) -- Realistic PR diffs: `benchmarks/real_diffs/` -- Example workflows: `examples/github_workflows/` -- Metrics: `scripts/collect_pilot_metrics.py`, `docs/benchmarks/adoption-summary.json` -- External pilot registry: `docs/benchmarks/external-pilots-registry.json` -- Self-report template: `docs/templates/external_pilot_report.template.json` -- Ingest script: `scripts/ingest_external_pilot_metrics.py` +## 5. Reporting and references -## What to report +Publish a study only with maintainer permission and clearly label it as *independent external*, *maintained-consumer*, or *dogfood*. The existing [external pilot registry](benchmarks/external-pilots-registry.json) includes maintained and internal examples; they are not independent customer validation. -- Repository name (link) -- Check type(s) enabled -- Advisory duration (dates) -- PRs evaluated / blocked / false positives -- Median `ovk check` latency -- Strict mode enabled (yes/no) +The adoption report and supporting paths are [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md), `scripts/collect_pilot_metrics.py`, `scripts/ingest_external_pilot_metrics.py`, `scripts/render_pilot_metrics.py`, and `docs/benchmarks/external-pilots-registry.json`. -Add entries to [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md). +Operational ownership and milestones are tracked in [North Star issue #216](https://github.com/fraware/open-verification-kernel/issues/216), with the first independent pilot in [issue #20](https://github.com/fraware/open-verification-kernel/issues/20). From 6310a16ceff435de5503dd2ba5e5ca726d6a5889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:41:37 -0700 Subject: [PATCH 039/104] integration(release): import #219 historical release corrections in docs/PILOT_CASE_STUDIES.md --- docs/PILOT_CASE_STUDIES.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/PILOT_CASE_STUDIES.md b/docs/PILOT_CASE_STUDIES.md index 8f145eb1a..0a190504a 100644 --- a/docs/PILOT_CASE_STUDIES.md +++ b/docs/PILOT_CASE_STUDIES.md @@ -2,7 +2,7 @@ Measured outcomes from example configurations in `examples/pilot_repos/`. Metrics use `ovk check` and `ovk verify` in advisory mode. -**Honesty:** in-repo pilot dogfood is **not** independent consumer validation and does not authorize `verified_source_sha` or `externally_calibrated_strict`. Live signed pin for external remotes remains `v1.2.1` until `v1.3.0-rc.1` is attributable. +**Honesty:** in-repo pilot dogfood is **not** independent consumer validation and does not authorize `verified_source_sha` or `externally_calibrated_strict`. The last published tag is `v1.2.1`, whose annotated Git tag was reported unsigned by GitHub on 2026-10-15; do not call it a Git-signed or protected pin. For any permitted external advisory experiment, use a reviewed **full 40-hex commit SHA** and independently verify applicable release assets and operator trust. ## In-repo reference pilot @@ -12,7 +12,7 @@ Simulates how an external repo would consume OVK before community pilots land. **Manifest:** `examples/pilot_repos/external_oss_ci_secrets.json` -**Consumer path:** local Action (`./`) with package version matching the workflow under test. Prefer mirroring `@v1.2.1` for live signed pins; bump only after an attributable RC tag. +**Consumer path:** local Action (`./`) with package version matching the workflow under test. Prefer the audited commit SHA `a27d5720f4350c00bca34f71d991c31f5a2f38c7` behind the published `v1.2.1` annotated tag for a legacy advisory Action example, with explicit owner review. This pins code identity but does not independently verify signer, source semantics, or dependency integrity. Bump only after an attributable new release. | Metric | Target | Measured | |--------|--------|----------| @@ -137,14 +137,14 @@ Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurement **Manifest:** adapted from `examples/pilot_repos/external_oss_ci_secrets.json` or `self_protection_only.json` -**Rollout:** advisory (at least 14 days) → strict on `main` +**Rollout:** advisory (at least 14 days) → independent review of qualified source coverage, sample adequacy, unknowns, false negatives, false positives, governance and rollback → optional separately approved strict deployment. There is no automatic promotion based on elapsed time or one metric. | Metric | Target | Measured | |--------|--------|----------| | Advisory period | ≥14 days | _TBD_ | | PRs scanned | — | _TBD_ | | Block rate on unsafe workflow diff | 100% | _TBD_ | -| False positive rate | <5% before strict | _TBD_ | +| False-positive numerator/denominator and uncertainty | report all cases; no automatic strict threshold from small N | _TBD_ | | Time to first green after repair | <5 minutes | _TBD_ | | Strict enable date | — | _TBD_ | | p95 Action latency | — | _TBD_ | @@ -153,6 +153,6 @@ Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurement ### Publication checklist - [ ] Advisory window ≥14 days with artifact retention -- [ ] False-positive rate computed and under 5% (or strict not yet enabled) -- [ ] Link to workflow file and version pin (`@v1.2.1` live; `@v1.3.0-rc.1` after attributable cut) +- [ ] Report false positives with denominator and sampling limitations; strict remains off until independent source-profile, sensitivity/false-negative and policy qualification +- [ ] Link to workflow file and exact immutable commit SHA; do not rely on unprotected release tags as immutable installation pins - [ ] Update the **Active and recruiting** table with measured values From 406c4f61a9af96db2aa2a45db66bed5ddeb75cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:41:40 -0700 Subject: [PATCH 040/104] integration(release): import #219 historical release corrections in docs/RELEASE.md --- docs/RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 2ccb800dc..5052abfb2 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -323,7 +323,7 @@ Consumers should trust only the Publish workflow path, exact immutable tag ident ## Historical release evidence -Historical releases such as `v1.2.1` remain immutable and retain their original evidence. Their workflow runs, release artifacts, and Sigstore identities are historical facts, not evidence for `v1.3.0-rc.1`. +Historical published releases such as `v1.2.1` retain their original release-artifact and workflow evidence, but **ref immutability is not established**. On 2026-10-15 GitHub reported the annotated `v1.2.1` tag as `unsigned`, the release's `immutable` field as `false`, and no active repository rulesets; the live `main` branch object reports `protected=false`. Verify each release asset, signed bundle, and immutable **commit SHA** separately. Historical workflows and Sigstore identities do not authorize `v1.3.0-rc.1`. Independent consumer repositories currently used by the consumer-pin workflow are: From e56b9bbe1cbda7cc4c434b42ec2c8f914f852f3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:41:43 -0700 Subject: [PATCH 041/104] integration(release): import #219 historical release corrections in docs/RELEASE_NOTES_v1.2.1.md --- docs/RELEASE_NOTES_v1.2.1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/RELEASE_NOTES_v1.2.1.md b/docs/RELEASE_NOTES_v1.2.1.md index ebded07a2..e6415be42 100644 --- a/docs/RELEASE_NOTES_v1.2.1.md +++ b/docs/RELEASE_NOTES_v1.2.1.md @@ -1,10 +1,10 @@ # OVK v1.2.1 -Patch release to close the immutable-tag keyless Sigstore production pin and fix release-gate regressions on `main`. +Historical patch release associated with keyless Sigstore artifact signing; Git-tag immutability was not established. Changes include release-gate regressions on `main`. ## Changes -- Publish workflow on an immutable `refs/tags/v1.2.1` Release: build → keyless cosign → verify → attach `*.cosign.bundle.json` +- Publish workflow associated with the published `v1.2.1` release: build → keyless cosign → verify → attach `*.cosign.bundle.json`. **Correction (2026-10-15):** the GitHub annotated tag object reports `verification.verified=false` (`unsigned`) and the release's `immutable` API field is `false`. Treat artifact Sigstore bundles separately from Git tag signing and ref protection. - Composite Action installs sync `ovk/package_data` before `pip install .` so policy schemas ship into site-packages - Align unit tests with honest CBMC template-harness limits, relative provenance URIs, and reliability-based repo-memory priors @@ -26,7 +26,7 @@ pip install -e '.[dev]' env: OVK_PACKAGE_VERSION: "1.2.1" steps: - - uses: fraware/open-verification-kernel@v1.2.1 + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 # exact historical source commit ``` ## Consumer Sigstore verify From d43d962a86217de593b5a6e9a81edc63f7ec13ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:42:02 -0700 Subject: [PATCH 042/104] integration(pilot): preserve read-only advisory fork workflow and disable check publication --- .../github_workflows/pilot_fork_adopter.yml | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/examples/github_workflows/pilot_fork_adopter.yml b/examples/github_workflows/pilot_fork_adopter.yml index aca8eb42a..a690c897c 100644 --- a/examples/github_workflows/pilot_fork_adopter.yml +++ b/examples/github_workflows/pilot_fork_adopter.yml @@ -2,7 +2,8 @@ name: OVK Pilot (Fork Adopter) # Copy to .github/workflows/ovk-pilot.yml in your repository. # Phase 0: advisory check + manifest verify; upload metrics for maintainer ingest. -# See docs/EXTERNAL_PILOT_PLAYBOOK.md for advisory → strict rollout. +# Advisory-only pilot. Strict enforcement is a separate qualification gate. +# See docs/EXTERNAL_PILOT_PLAYBOOK.md for supported claims and reviewer protocol. on: pull_request: @@ -10,20 +11,22 @@ on: permissions: contents: read + pull-requests: read env: - # Historical v1.2.1 distribution. This old Action has a fallback to checkout - # if package installation fails; pinning does not establish package-byte identity. - # For consenting advisory evaluations only; see release status before use. + # Historical v1.2.1 Action has fallback to checkout if package installation fails. + # The full source SHA below pins only the top-level Action, not its transitive dependencies. + # Consent-based, advisory use only: its manually supplied diff is not source-bound. OVK_PACKAGE_VERSION: "1.2.1" jobs: ovk-ci-secrets-pilot: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 + persist-credentials: false - name: Build PR diff for OVK run: | git fetch origin "${{ github.base_ref }}" @@ -34,6 +37,7 @@ jobs: mode: advisory use-check: "true" changed-files: ovk-pr.diff + emit-check: "false" post-comment: "false" - name: OVK advisory verify (ci_secrets lane manifest) uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 @@ -41,14 +45,14 @@ jobs: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json bundle-output-dir: ovk-pilot-bundle + emit-check: "false" post-comment: "false" - name: Upload pilot metrics artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: ovk-pilot-artifacts path: | ovk-evidence.json ovk-pilot-bundle/** - external_pilot_report.json retention-days: 30 - if-no-files-found: ignore + if-no-files-found: warn From 5c400f28ea40b1915ec01c25134c05d9983afef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:42:30 -0700 Subject: [PATCH 043/104] docs(security): make README copyable Action example read-only and source-pinned --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 15bbd5b5b..4dd40196f 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ Installation, package, optional-solver, and signing paths: [docs/INTEGRATION.md] ## GitHub Actions -Add OVK to a pull request workflow. Start in **advisory** mode (reports findings without failing the job); move to **strict** only for lanes/profiles whose signal and trusted inputs you have calibrated for that repository. +Add OVK to a pull request workflow in **advisory** mode, with the repository maintainer's consent and no merge-blocking authority. The current public source profiles lack independent strict qualification; strict enforcement is a separate release, evidence, and source-model security decision. ```yaml name: Verify PR @@ -141,24 +141,25 @@ on: permissions: contents: read - pull-requests: write - checks: write # required when emit-check: true jobs: ovk: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 2 + persist-credentials: false # SHA of the latest published v1.2.1 release's direct source commit. # Change this only after reviewing the release identity and limits. - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 env: OVK_PACKAGE_VERSION: "1.2.1" with: - mode: advisory # switch to strict only after calibration - use-check: "true" # analyze the PR diff automatically - emit-check: "true" # optional; requires checks: write - post-comment: "true" # requires pull-requests: write + mode: advisory # reviewed, non-blocking compatibility pilot + use-check: "true" # legacy automatic PR-diff path; not a signed receipt + emit-check: "false" # no check-write authority + post-comment: "false" # no comment-write authority ``` **Release safety boundary (verified 2026-10-15):** `v1.3.0-rc.1` is not published or tagged. This example runs **historical `v1.2.1` code**, pinned to its original source commit; it does not include the draft guarantee-continuity feature or the new fail-closed installer. **The `v1.2.1` Action falls back to its checkout if the requested PyPI version fails to install.** Do not treat that installation as independently authenticated solely because `OVK_PACKAGE_VERSION` is set. The old Git tag is unsigned, and its GitHub Release is not marked immutable. The corrected installation contract in draft PR #220 takes effect only in a later, independently authorized release. Until then use the checkout examples for development or a separately governed, consenting **advisory-only** experiment. See [release status](docs/CURRENT_RELEASE_STATUS.md), [pilot playbook](docs/EXTERNAL_PILOT_PLAYBOOK.md), and [launch checklist](docs/DEVELOPER_PREVIEW_READINESS.md). From 0dfec6df542ac089123c5f1cf4ba95118076b77b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:42:54 -0700 Subject: [PATCH 044/104] docs(security): distinguish historical write-enabled examples from authorized advisory pilots --- docs/INTEGRATION.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index 3e4516ff2..23922a9fd 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -30,6 +30,8 @@ Optional Z3: `pip install -e '.[solvers]'` Optional HMAC signing: `export OVK_SIGNING_KEY=your-signing-secret` +**Signing trust boundary:** `OVK_SIGNING_KEY` is a shared HMAC secret used for signing and verifying; it is not a public verification key. Do not expose it to fork-originated `pull_request` jobs, upload it with evidence artifacts, or mistake a locally HMAC-signed bundle for independently authenticated source or release provenance. + ## First demo Self-protection failing case (gate removed): @@ -95,6 +97,8 @@ permissions: ### Single check type (self-protection) +The following historical illustration includes optional check/comment publishing with write privileges. **Do not copy it as the first external pilot workflow.** For consenting adopters, use the [read-only advisory example](../examples/github_workflows/external_consumer.yml) and the [pilot playbook](EXTERNAL_PILOT_PLAYBOOK.md). + ```yaml name: OVK on: @@ -171,9 +175,9 @@ Downstream example: mode: advisory use-check: "true" emit-check: "true" -- name: Fail when OVK blocks +- name: Surface OVK block for human review if: steps.ovk.outputs.recommendation == 'block' - run: exit 1 + run: echo "OVK reported a block; inspect scope and supporting evidence" ``` **Artifact files:** @@ -256,16 +260,18 @@ Required GitHub check name: **`Open Verification Kernel`** (from `ovk/core/githu ### Rollout stages +**Stages 2–4 are future, separately qualified configurations**, not a current promotion path. Current public source profiles are advisory, the repository's protection and release-signing gates are open, and small in-repo samples do not authorize strict enforcement. + | Stage | Workflow example | Mode | Publish check run | Branch protection | |---|---|---|---|---| | 1 — Advisory only | `examples/github_workflows/pilot_fork_adopter.yml` | advisory | no | none | | 2 — Advisory + visibility | `examples/github_workflows/pilot_advisory_with_comment.yml` | advisory | yes | optional | -| 3 — Strict enforcement | `examples/github_workflows/external_consumer.yml` | strict | yes | required on `main` | +| 3 — Strict enforcement (future) | No currently approved copyable workflow | strict | qualified-only | approved protection | | 4 — Required check wired | `examples/github_workflows/branch_protection_required_check.yml` | strict | yes | required check | -Steps for strict enforcement: +Steps for strict enforcement **only following source-profile, release, independent adjudication, owner approval and rollback qualification**: -1. Run strict + `emit-check: true` on a sample PR and confirm the check appears on the commit. +1. On the separately authorized repository and exact source profile, test strict + `emit-check: true` and confirm correct source binding and check attribution. 2. In repository **Settings → Branches**, add **`Open Verification Kernel`** as a required status check on protected branches. 3. Supply explicit `check-metadata` when verifying gate preservation (auto-collected metadata is best-effort only). @@ -273,6 +279,8 @@ Steps for strict enforcement: ## External repository smoke test +The following is a **historical, write-enabled integration example**, not a copyable first-user or fork-safe installation. The default external pilot uses the read-only example in the playbook. + Create `.github/workflows/ovk-smoke.yml` in an integration repository: ```yaml From ee150969397397c79a780f0cf63d6eab24149604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:43:25 -0700 Subject: [PATCH 045/104] fix(pilot): invalidate prior reviewer error rates on every new artifact cohort --- scripts/ingest_external_pilot_metrics.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/ingest_external_pilot_metrics.py b/scripts/ingest_external_pilot_metrics.py index 9ba5c3b44..596194216 100644 --- a/scripts/ingest_external_pilot_metrics.py +++ b/scripts/ingest_external_pilot_metrics.py @@ -143,13 +143,13 @@ def build_pilot_row( row.update({key: value for key, value in existing.items() if value is not None or key in row}) row["repository"] = repo if artifacts_dir is not None and artifacts_dir.is_dir(): + # An explicitly ingested artifact directory constitutes a new, + # potentially unreviewed cohort. Even a single evidence bundle or an + # incomplete aggregate must not inherit older human error labels. + # Cohort equality has not been independently authenticated. + row["false_positives"] = None + row["false_positive_rate"] = None artifact_metrics = _metrics_from_artifacts(artifacts_dir) - if artifact_metrics.get("prs_evaluated") is not None: - # A new artifact cohort invalidates older adjudicated error - # counts. Avoid carrying forward a historical zero as evidence - # that the current, unreviewed cohort had no false positives. - row["false_positives"] = None - row["false_positive_rate"] = None row.update({key: value for key, value in artifact_metrics.items() if value is not None}) if report_path is not None and report_path.exists(): # A reviewer-provided report is the only source of error labels. From f332f63efa58e6cb9c3d88b21b0d9bfd76369510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:43:38 -0700 Subject: [PATCH 046/104] test(pilot): prevent stale error-rate reuse on incomplete artifact cohorts --- tests/test_ingest_external_pilot_metrics.py | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index 28d562908..c295ed75e 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -214,6 +214,47 @@ def test_new_artifacts_clear_stale_external_error_labels( assert row["false_positive_rate"] is None +@pytest.mark.parametrize( + "new_artifact_metrics", + [{"prs_blocked": 1}, {"median_check_latency_ms": 23.0}, {}], +) +def test_partial_new_artifacts_invalidate_stale_error_labels( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, new_artifact_metrics: dict +) -> None: + """No denominator or aggregate results must not imply zero new errors.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: new_artifact_metrics, + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, artifacts_dir=tmp_path) + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + + +def test_without_new_artifacts_historical_labels_are_not_silently_rewritten() -> None: + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 1, + "false_positive_rate": 0.5, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing) + assert row["false_positives"] == 1 + assert row["false_positive_rate"] == 0.5 + + def test_external_report_supplies_separate_reviewed_labels( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 55f7d60331110dc16179da0526e606ea02255aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:43:59 -0700 Subject: [PATCH 047/104] docs(pilot): flag legacy unbound diffs and prohibit automatic publication --- docs/EXTERNAL_PILOT_PLAYBOOK.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/EXTERNAL_PILOT_PLAYBOOK.md b/docs/EXTERNAL_PILOT_PLAYBOOK.md index 69b45b3ef..a01bce36c 100644 --- a/docs/EXTERNAL_PILOT_PLAYBOOK.md +++ b/docs/EXTERNAL_PILOT_PLAYBOOK.md @@ -52,6 +52,7 @@ jobs: mode: advisory use-check: "true" changed-files: ovk-pr.diff + emit-check: "false" post-comment: "false" - name: OVK advisory verification uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 @@ -59,6 +60,7 @@ jobs: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json bundle-output-dir: ovk-pilot-bundle + emit-check: "false" post-comment: "false" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 if: always() @@ -71,7 +73,7 @@ jobs: if-no-files-found: warn ``` -Review the actual code and external actions before enabling this workflow. The full source-commit SHA pins the top-level Action, yet the historical Action may reference transitive dependencies by mutable tags; independently review that supply chain and keep the pilot non-blocking. The check stays advisory, cannot authorize a merge, and does not receive secrets from untrusted PRs. +Review the actual code and external actions before enabling this workflow. The full source-commit SHA pins the top-level Action, yet the historical Action may reference transitive dependencies by mutable tags and may fall back to installing the Action checkout if its requested distribution version is unavailable. Independently review that supply chain and keep the pilot non-blocking. The legacy caller-supplied diff is **unbound source input**, not an authenticated verified-PR receipt; the workflow explicitly disables both check publication and comments. The check stays advisory, cannot authorize a merge, and does not receive secrets from untrusted PRs. ## 2. Observe all naturally occurring PRs @@ -95,7 +97,7 @@ python scripts/render_pilot_metrics.py \ The optional report starts from [external_pilot_report.template.json](templates/external_pilot_report.template.json); replace its placeholders. Even if a file named `external_pilot_report.json` appears within a downloaded CI artifact directory, the ingester **does not automatically adopt its adjudication fields**. A human operator must deliberately select a separately reviewed report via `--report`, with the same repository identifier as `--repo`. Such reports remain self-reported until independently audited. `prs_evaluated`, `prs_blocked`, and latency can come from workflow artifacts. **False positives require separate human or independently specified ground-truth adjudication.** OVK's own `passed`, `merge_recommendation`, and checker `PASS/FAIL` fields are not ground-truth labels. The metric ingester must not manufacture a false-positive rate from them. -Only supply `false_positives` and `false_positive_rate` if a reviewer actually classified cases and the denominator and decisions are documented in the report's linked evidence or notes. Otherwise leave both `null`. If a new artifact cohort arrives without fresh adjudication, previously reported rates are stale and must not be carried forward as current measurements. A reviewer-provided number is still **self-reported** until independently audited. +Only supply `false_positives` and `false_positive_rate` if a reviewer actually classified cases and the denominator and decisions are documented in the report's linked evidence or notes. Otherwise leave both `null`. Every explicitly ingested artifact directory is treated as a potentially new, unreviewed cohort, including directories containing only one evidence bundle or no recognized aggregate. Previously reported error counts/rates are invalidated unless an explicit report supplies fresh adjudication. Cohort identity is not authenticated in this development ingester; operators must document report-to-artifact correspondence. Missing input must never be treated as zero observed errors. A reviewer-provided number is still **self-reported** until independently audited. Explicitly distinguish: - **Check/recommendation rate:** what OVK returned on actual PRs. From fa3f630772a2257ee2a10d9aed4fe0208852be23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:44:24 -0700 Subject: [PATCH 048/104] test(integration): enforce pilot metric placeholders and least-privilege release truth --- ...test_pilot_release_integration_contract.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/test_pilot_release_integration_contract.py diff --git a/tests/test_pilot_release_integration_contract.py b/tests/test_pilot_release_integration_contract.py new file mode 100644 index 000000000..c65092490 --- /dev/null +++ b/tests/test_pilot_release_integration_contract.py @@ -0,0 +1,80 @@ +"""Cross-PR contract for a non-blocking, attributable pilot entry point. + +This guards copyable workflow safety and reporting provenance. It is not an +independent assessment of the pilot's scientific accuracy or adopter identity. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +HISTORICAL_ACTION = "fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7" + + +def test_fork_adopter_example_has_no_publication_or_write_token() -> None: + workflow_path = ROOT / "examples/github_workflows/pilot_fork_adopter.yml" + raw = workflow_path.read_text(encoding="utf-8") + workflow = yaml.safe_load(raw) + permissions = workflow["permissions"] + assert permissions["contents"] == "read" + assert "checks" not in permissions + assert permissions.get("pull-requests", "read") == "read" + + steps = workflow["jobs"]["ovk-ci-secrets-pilot"]["steps"] + checkouts = [s for s in steps if s.get("uses", "").startswith("actions/checkout@")] + assert len(checkouts) == 1 + assert checkouts[0]["with"]["persist-credentials"] is False + assert re.fullmatch(r"actions/checkout@[0-9a-f]{40}", checkouts[0]["uses"]) + + ovk_steps = [s for s in steps if s.get("uses", "").startswith("fraware/open-verification-kernel@")] + assert len(ovk_steps) == 2 + for step in ovk_steps: + assert step["uses"] == HISTORICAL_ACTION + assert step["with"]["mode"] == "advisory" + assert step["with"]["emit-check"] == "false" + assert step["with"]["post-comment"] == "false" + + assert ovk_steps[0]["with"]["changed-files"] == "ovk-pr.diff" + assert "unbound source input" in (ROOT / "docs/EXTERNAL_PILOT_PLAYBOOK.md").read_text(encoding="utf-8") + assert "external_pilot_report.json" not in raw + + +def test_copyable_readme_example_is_least_privilege() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + section = readme.split("## GitHub Actions", 1)[1].split("## Built for agents", 1)[0] + assert "contents: read" in section + assert "checks: write" not in section + assert "pull-requests: write" not in section + assert HISTORICAL_ACTION in section + assert 'emit-check: "false"' in section + assert 'post-comment: "false"' in section + assert "falls back to its checkout" in section + + +def test_pilot_template_never_invents_measured_zeroes() -> None: + report = json.loads( + (ROOT / "docs/templates/external_pilot_report.template.json").read_text(encoding="utf-8") + ) + for field in ( + "advisory_start", "advisory_end", "prs_evaluated", "prs_blocked", + "false_positives", "false_positive_rate", "median_check_latency_ms", + ): + assert report[field] is None, field + assert report["strict_enabled"] is False + + +def test_release_docs_and_strict_example_are_explicitly_qualified() -> None: + status = (ROOT / "docs/CURRENT_RELEASE_STATUS.md").read_text(encoding="utf-8") + integration = (ROOT / "docs/INTEGRATION.md").read_text(encoding="utf-8") + assert "protected=false" in status + assert "verification.verified=false" in status + assert "immutable=false" in status + assert "v1.3.0-rc.1" in status + assert "shared HMAC secret" in integration + assert "No currently approved copyable workflow" in integration From 948f36ce07dad99c565060c63ffcd3c5401e1fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:46:09 -0700 Subject: [PATCH 049/104] fix(metrics): never infer false-positive rate from checker fixture verdicts --- scripts/collect_pilot_metrics.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/scripts/collect_pilot_metrics.py b/scripts/collect_pilot_metrics.py index ce9674f08..cb1cc4e4f 100644 --- a/scripts/collect_pilot_metrics.py +++ b/scripts/collect_pilot_metrics.py @@ -54,16 +54,6 @@ def _bundle_artifacts(bundle_dir: Path | None) -> list[str]: return sorted(path.name for path in bundle_dir.rglob("*") if path.is_file()) -def _false_positive_rate(results: list[dict[str, Any]]) -> float | None: - evaluated = [item for item in results if "passed" in item] - if not evaluated: - return None - false_positives = sum( - 1 for item in evaluated if not item.get("passed") and item.get("merge_recommendation") == "block" - ) - return false_positives / len(evaluated) - - def _median_elapsed_ms(results: list[dict[str, Any]]) -> float | None: timings = [float(item["elapsed_ms"]) for item in results if "elapsed_ms" in item] if not timings: @@ -190,7 +180,7 @@ def collect_pilot_metrics( "manifests_passed": manifests_passed, "manifests_total": manifests_total, "median_elapsed_ms": median_elapsed, - "false_positive_rate": _false_positive_rate(results), + "false_positive_rate": None, # no independent ground-truth adjudication in fixture output "block_rate_on_unsafe_fixture": block_rate, "external_manifest": _repo_relative(manifest_path), }, From c866d5717fd8f9ef900f362ad0a6b62796d07fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:46:16 -0700 Subject: [PATCH 050/104] fix(metrics): prevent reimport of historical unadjudicated dogfood error rates --- scripts/render_pilot_metrics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/render_pilot_metrics.py b/scripts/render_pilot_metrics.py index d99b4f9a1..31c5b78be 100644 --- a/scripts/render_pilot_metrics.py +++ b/scripts/render_pilot_metrics.py @@ -122,7 +122,9 @@ def render_adoption_summary( "manifests_passed": pilot_dogfood.get("manifests_passed"), "manifests_total": pilot_dogfood.get("manifests_total"), "median_elapsed_ms": pilot_dogfood.get("median_elapsed_ms"), - "false_positive_rate": pilot_dogfood.get("false_positive_rate"), + # Legacy fixture PASS/BLOCK is not independent ground truth. + # No adjudication provenance is accepted on this dogfood path. + "false_positive_rate": None, "external_manifest": pilot_dogfood.get( "external_manifest", "examples/pilot_repos/external_oss_ci_secrets.json" ), From 5e43290cc383c738ba5c788c2f7737f718e10d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:46:31 -0700 Subject: [PATCH 051/104] test(metrics): fixture BLOCK cannot be adjudicated as false positive --- tests/test_collect_pilot_metrics.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_collect_pilot_metrics.py b/tests/test_collect_pilot_metrics.py index 55e551921..559e2be84 100644 --- a/tests/test_collect_pilot_metrics.py +++ b/tests/test_collect_pilot_metrics.py @@ -16,7 +16,27 @@ def test_collect_pilot_metrics_schema_valid() -> None: require_schema_valid(metrics, schema, context="pilot metrics") validate_metrics(metrics) assert metrics["pilot_report"]["manifests_passed"] == metrics["pilot_report"]["manifests_total"] - assert metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] == 0.0 + assert metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] is None + + +def test_checker_block_on_fixture_does_not_qualify_false_positive_rate(tmp_path: Path) -> None: + """A failed fixture and a BLOCK are both tool-relative signals, not ground truth.""" + report = tmp_path / "pilot-report.json" + report.write_text( + json.dumps({ + "results": [ + {"passed": False, "merge_recommendation": "block", "elapsed_ms": 12.0}, + {"passed": True, "merge_recommendation": "allow", "elapsed_ms": 14.0}, + ], + "manifests_total": 2, + "manifests_passed": 1, + }), + encoding="utf-8", + ) + metrics = collect_pilot_metrics(source="pilot-dogfood", pilot_report_path=report) + validate_metrics(metrics) + assert metrics["adoption"]["pilot_dogfood"]["manifests_total"] == 2 + assert metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] is None def test_collect_pilot_metrics_with_evidence(tmp_path: Path) -> None: From 788c0772c4a160fd5a2c35f61af807521884cecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:46:39 -0700 Subject: [PATCH 052/104] test(metrics): renderer rejects legacy self-judged zero-error rates --- tests/test_render_pilot_metrics.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_render_pilot_metrics.py b/tests/test_render_pilot_metrics.py index 7b48d3a81..c212bb686 100644 --- a/tests/test_render_pilot_metrics.py +++ b/tests/test_render_pilot_metrics.py @@ -28,6 +28,15 @@ def test_render_adoption_summary_shape() -> None: assert summary["pilot_dogfood"]["manifests_passed"] == metrics["adoption"]["pilot_dogfood"]["manifests_passed"] +def test_renderer_discards_unadjudicated_legacy_fixture_rate() -> None: + """A stale aggregate must not recreate a self-judged 0% error rate.""" + metrics = collect_pilot_metrics(source="local") + metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] = 0.0 + rendered = render_adoption_summary(metrics) + validate_summary(rendered) + assert rendered["pilot_dogfood"]["false_positive_rate"] is None + + def test_adoption_summary_template_is_schema_valid() -> None: template = read_json_file(Path("docs/benchmarks/adoption-summary.json")) schema = read_json_file(Path("schemas/adoption.summary.schema.json")) From ddda0e4300fc5391ffe35716325acd685c285a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:47:17 -0700 Subject: [PATCH 053/104] data(pilot): remove unadjudicated zero-error claims from maintained fixtures --- docs/benchmarks/external-pilots-registry.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/benchmarks/external-pilots-registry.json b/docs/benchmarks/external-pilots-registry.json index 01a6beb62..750f5d87b 100644 --- a/docs/benchmarks/external-pilots-registry.json +++ b/docs/benchmarks/external-pilots-registry.json @@ -1,6 +1,6 @@ { "schema_version": "ovk.external_pilots_registry.v1", - "updated_at": "2026-07-25T17:08:12Z", + "updated_at": "2026-10-15T23:46:39Z", "external_pilots": [ { "repository": "fraware/ovk-consumer-fastapi-terraform", @@ -13,14 +13,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 95.62, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/fastapi-terraform/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "fraware/ovk-consumer-express-actions", @@ -33,14 +33,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 81.71, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/express-actions/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "in-repo/ovk-pilot-infra-terraform-k8s", @@ -53,14 +53,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 3, "prs_blocked": 2, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 46.14, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": "docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml", "evidence_url": "docs/pilots/infra-terraform-k8s/pilot-report.json", - "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md." + "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "TBD - recruiting first true external OSS adopter", From 1699d64212bba336acd87a419fe7b4c14638fabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:47:25 -0700 Subject: [PATCH 054/104] data(adoption): mark unadjudicated dogfood false-positive figures as unknown --- docs/benchmarks/adoption-summary.json | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/benchmarks/adoption-summary.json b/docs/benchmarks/adoption-summary.json index c7ad230ba..861eab86a 100644 --- a/docs/benchmarks/adoption-summary.json +++ b/docs/benchmarks/adoption-summary.json @@ -1,47 +1,47 @@ { "schema_version": "ovk.adoption_summary.v1", "ovk_version": "1.2.1", - "updated_at": "2026-07-25T17:11:57Z", + "updated_at": "2026-10-15T23:46:39Z", "formal_pr_bench": { "cases_total": 132, "cases_passed": 132, - "pass_rate": 1.0, - "intent_recall": 1.0, + "pass_rate": 1, + "intent_recall": 1, "by_category": { "adversarial": { "cases_total": 2, "cases_passed": 2, - "pass_rate": 1.0 + "pass_rate": 1 }, "intent_recall": { "cases_total": 3, "cases_passed": 3, - "pass_rate": 1.0 + "pass_rate": 1 }, "lane": { "cases_total": 100, "cases_passed": 100, - "pass_rate": 1.0 + "pass_rate": 1 }, "multi_backend": { "cases_total": 1, "cases_passed": 1, - "pass_rate": 1.0 + "pass_rate": 1 }, "real_diff": { "cases_total": 18, "cases_passed": 18, - "pass_rate": 1.0 + "pass_rate": 1 }, "repair_loop": { "cases_total": 4, "cases_passed": 4, - "pass_rate": 1.0 + "pass_rate": 1 }, "routing": { "cases_total": 4, "cases_passed": 4, - "pass_rate": 1.0 + "pass_rate": 1 } }, "timing_ms": { @@ -50,14 +50,14 @@ "max": 34.02709299999884 } }, - "real_diff_recall": 1.0, + "real_diff_recall": 1, "pilot_dogfood": { "source": "local", "last_run": "2026-07-25T17:11:57Z", "manifests_passed": 8, "manifests_total": 8, "median_elapsed_ms": 4.71550000111165, - "false_positive_rate": 0.0, + "false_positive_rate": null, "external_manifest": "examples/pilot_repos/external_oss_ci_secrets.json", "weekly_schedule": "0 7 * * 1", "workflow": ".github/workflows/pilot-dogfood.yml", @@ -75,14 +75,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 95.62, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/fastapi-terraform/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "fraware/ovk-consumer-express-actions", @@ -95,14 +95,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 81.71, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/express-actions/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "in-repo/ovk-pilot-infra-terraform-k8s", @@ -115,14 +115,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 3, "prs_blocked": 2, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 46.14, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": "docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml", "evidence_url": "docs/pilots/infra-terraform-k8s/pilot-report.json", - "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md." + "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "TBD - recruiting first true external OSS adopter", From 6a2e25eaba3910f3d06e529534239c1c5b25235d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:47:42 -0700 Subject: [PATCH 055/104] docs(pilot): correct circular fixture false-positive claims and strict gate --- docs/PILOT_CASE_STUDIES.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/PILOT_CASE_STUDIES.md b/docs/PILOT_CASE_STUDIES.md index 0a190504a..13fdd5fdd 100644 --- a/docs/PILOT_CASE_STUDIES.md +++ b/docs/PILOT_CASE_STUDIES.md @@ -2,7 +2,7 @@ Measured outcomes from example configurations in `examples/pilot_repos/`. Metrics use `ovk check` and `ovk verify` in advisory mode. -**Honesty:** in-repo pilot dogfood is **not** independent consumer validation and does not authorize `verified_source_sha` or `externally_calibrated_strict`. The last published tag is `v1.2.1`, whose annotated Git tag was reported unsigned by GitHub on 2026-10-15; do not call it a Git-signed or protected pin. For any permitted external advisory experiment, use a reviewed **full 40-hex commit SHA** and independently verify applicable release assets and operator trust. +**Honesty:** in-repo pilot dogfood is **not** independent consumer validation and does not authorize `verified_source_sha` or `externally_calibrated_strict`. Fixture pass/block output does not establish a false-positive rate; unadjudicated error rates are reported as unknown. The last published tag is `v1.2.1`, whose annotated Git tag was reported unsigned by GitHub on 2026-10-15; do not call it a Git-signed or protected pin. For any permitted external advisory experiment, use a reviewed **full 40-hex commit SHA** and independently verify applicable release assets and operator trust. ## In-repo reference pilot @@ -19,7 +19,7 @@ Simulates how an external repo would consume OVK before community pilots land. | Advisory mode | yes | yes | | Unsafe workflow diff block rate | 100% | _from weekly artifact_ | | Safe manifest pass rate | 100% | _from weekly artifact_ | -| False positive rate (fixtures) | 0% | _from pilot-metrics.json_ | +| Adjudicated false-positive rate | — | Unknown; the fixture outcomes contain no independent reviewer labels | | Median manifest latency | — | _from pilot-metrics.json_ | | Adoption summary | published | `docs/benchmarks/adoption-summary.json` | @@ -45,7 +45,7 @@ python scripts/render_pilot_metrics.py --metrics pilot-dogfood-report.json |--------|--------| | Time to first green run | ~0.3s | | Check types run | 1 | -| False positive rate (fixtures) | 0% (1/1 pass case) | +| Adjudicated false-positive rate | Unknown; one passing fixture is not independent error adjudication | Lowest-friction path for repos adding agent-authored PR gates. @@ -67,7 +67,7 @@ Lowest-friction path for repos adding agent-authored PR gates. |--------|--------| | Time to green (all pass fixtures) | ~2.1s | | Check types run | 5 | -| False positive rate (fixtures) | 0% (5/5 pass cases) | +| Adjudicated false-positive rate | Unknown; five passing fixtures are not independent error adjudication | | Artifacts | evidence, markdown, attestation, manifest, quality report | ## Cedar, TLA+, and Kani backends @@ -124,12 +124,12 @@ Source of truth: [external-pilots-registry.json](benchmarks/external-pilots-regi | Repository | Status | Kind | Check types | Advisory period | False positive rate | Strict enabled | |-------|--------|------|-------------|-----------------|---------------------|----------------| -| [fraware/ovk-consumer-fastapi-terraform](pilots/fastapi-terraform/REPORT.md) | advisory | Maintained consumer | ci_secrets, infrastructure | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | -| [fraware/ovk-consumer-express-actions](pilots/express-actions/REPORT.md) | advisory | Maintained consumer | ci_secrets, self_protection | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | -| [in-repo/ovk-pilot-infra-terraform-k8s](pilots/infra-terraform-k8s/REPORT.md) | advisory | In-repo maintained profile | infrastructure, ci_secrets | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | +| [fraware/ovk-consumer-fastapi-terraform](pilots/fastapi-terraform/REPORT.md) | advisory | Maintained consumer | ci_secrets, infrastructure | 2026-07-11 – 2026-07-25 | unknown (unadjudicated fixtures) | no | +| [fraware/ovk-consumer-express-actions](pilots/express-actions/REPORT.md) | advisory | Maintained consumer | ci_secrets, self_protection | 2026-07-11 – 2026-07-25 | unknown (unadjudicated fixtures) | no | +| [in-repo/ovk-pilot-infra-terraform-k8s](pilots/infra-terraform-k8s/REPORT.md) | advisory | In-repo maintained profile | infrastructure, ci_secrets | 2026-07-11 – 2026-07-25 | unknown (unadjudicated fixtures) | no | | TBD - recruiting first true external OSS adopter | recruiting | True external OSS (open) | ci_secrets | — | — | no | -Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurements with `strict_mode_recommendation: remain_advisory`. They do not replace a true independent external OSS adopter. When such a repo completes advisory rollout, ingest artifacts, update the registry, re-render the adoption summary, and keep the kind label explicit. Target: under 5% false positives before enabling strict mode on protected branches. +Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurements with `strict_mode_recommendation: remain_advisory`. They do not replace a true independent external OSS adopter. When such a repo completes advisory rollout, ingest artifacts, update the registry, re-render the adoption summary, and keep the kind label explicit. No numeric fixture-rate threshold authorizes strict enforcement. The formerly displayed 0.0 error rates were not independently adjudicated and are now null in the registry and adoption summary; historical values remain in Git history. An independent pilot needs explicit, reviewed false-positive and false-negative denominators, coverage and unknown handling. ### Reporting template From 59fd9ab3d293204ab700de28dd6f8d58886b74f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:48:07 -0700 Subject: [PATCH 056/104] test(pilot): guard archived summary against invented zero-error metrics --- ...test_pilot_release_integration_contract.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_pilot_release_integration_contract.py b/tests/test_pilot_release_integration_contract.py index c65092490..88fc23b1f 100644 --- a/tests/test_pilot_release_integration_contract.py +++ b/tests/test_pilot_release_integration_contract.py @@ -78,3 +78,31 @@ def test_release_docs_and_strict_example_are_explicitly_qualified() -> None: assert "v1.3.0-rc.1" in status assert "shared HMAC secret" in integration assert "No currently approved copyable workflow" in integration + + +def test_published_fixture_error_rates_are_unadjudicated_unknown() -> None: + """Corrected archived aggregates must not advertise 0% without ground truth.""" + registry = json.loads( + (ROOT / "docs/benchmarks/external-pilots-registry.json").read_text(encoding="utf-8") + ) + summary = json.loads( + (ROOT / "docs/benchmarks/adoption-summary.json").read_text(encoding="utf-8") + ) + assert summary["pilot_dogfood"]["false_positive_rate"] is None + + for data in (registry, summary): + maintained = [ + row for row in data["external_pilots"] + if "Maintained consumer" in row.get("notes", "") + or "In-repo maintained" in row.get("notes", "") + ] + assert len(maintained) == 3 + for row in maintained: + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + assert "not independently" in row["notes"] or "no independent" in row["notes"] + + study = (ROOT / "docs/PILOT_CASE_STUDIES.md").read_text(encoding="utf-8") + assert "0.0 (fixtures)" not in study + assert "Target: under 5%" not in study + assert "reported as unknown" in study From 2f50861df88a2ea9955cff41cff3a28138b58a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:50:57 -0700 Subject: [PATCH 057/104] docs(pilot): distinguish historical fixture outcomes from adjudicated error rates --- docs/pilots/fastapi-terraform/REPORT.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/pilots/fastapi-terraform/REPORT.md b/docs/pilots/fastapi-terraform/REPORT.md index 53c677628..76cb64080 100644 --- a/docs/pilots/fastapi-terraform/REPORT.md +++ b/docs/pilots/fastapi-terraform/REPORT.md @@ -8,12 +8,15 @@ Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +**Historical-data interpretation correction (2026-10-15):** The original companion JSON files preserve fixture counts and earlier self-reported error fields for historical inspection. Those fields have no independently adjudicated ground truth; the normalized external-pilot registry and adoption summary now record the rates as **null/unknown**. This clarification is not a new measurement. + ## Repository profile | Field | Value | |---|---| | Stack | FastAPI web app with Terraform infrastructure | -| Role | Independent maintained consumer gate (program section 23) | +| Role | Maintained consumer fixture validation (not independent adoption) | | Advisory workflow | `.github/workflows/ovk-advisory-pr.yml` | | Pilot ledger | Consumer `pilot/ledger.json` (automated_scenario rows only) | | Production gate | `production_gate_met: false` (no 30 human adjudications yet) | @@ -40,8 +43,8 @@ Documented playbook path reproduced locally against the consumer clone: | Metric | Measured | |---|---| -| False positives | **0** | -| False positive rate | **0.0** (0/2 fixture cases) | +| Independently adjudicated false positives | **Unknown** — fixture decisions lack independent reviewer ground truth | +| Independently adjudicated false-positive rate | **Unknown** — 2 fixtures do not estimate an external error rate | Passing workflow fixture was allowed; no incorrect blocks on the known-good case. @@ -49,7 +52,7 @@ Passing workflow fixture was allowed; no incorrect blocks on the known-good case | Metric | Measured | |---|---| -| False negatives | **0** | +| Independently adjudicated false negatives | **Unknown** — only specified known-bad fixtures were exercised | Known-bad secrets-on-`pull_request` fixture was blocked as expected (100% block rate on that unsafe fixture). @@ -73,13 +76,13 @@ No live PR adjudications yet — unknown appropriateness for production traffic ## Configuration changes -- Pin Action to `fraware/open-verification-kernel@v1.2.1` (immutable tag / audited SHA only). +- Historical source identity: `fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7`. The v1.2.1 tag was unsigned and unprotected in the 2026-10-15 audit; the original workflow used `@v1.2.1` and is an archival example, not an approved immutable installation. - Advisory mode via `.verification/config.yml` (`mode: advisory`, `default_on_unknown: require_human_review`). - Pilot manifest under `.verification/ci_secrets_pilot.json` (consumer) mirrored by in-repo `examples/pilot_repos/fastapi_terraform_consumer.json`. - Artifact upload of evidence / comment outputs for ingest. ## Strict-mode recommendation -**Remain advisory (`remain_advisory`).** Fixture FP rate is under 5%, but this publication is maintained-consumer dogfood without human-adjudicated live PRs. Do not enable strict required checks on protected branches from this evidence alone. +**Remain advisory (`remain_advisory`).** Known-good and known-bad fixture decisions matched their expected behavior, but no independently adjudicated live PR cohort or external false-positive rate exists. Strict required checks remain unauthorized. Median `ovk check` latency on fixture diffs: ~96 ms. From 070ae829b7d8e9fb0f4160341750378d4a8a228f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:51:01 -0700 Subject: [PATCH 058/104] docs(pilot): distinguish historical fixture outcomes from adjudicated error rates --- docs/pilots/express-actions/REPORT.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/pilots/express-actions/REPORT.md b/docs/pilots/express-actions/REPORT.md index 094a29d98..1a530a99d 100644 --- a/docs/pilots/express-actions/REPORT.md +++ b/docs/pilots/express-actions/REPORT.md @@ -8,12 +8,15 @@ Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +**Historical-data interpretation correction (2026-10-15):** The original companion JSON files preserve fixture counts and earlier self-reported error fields for historical inspection. Those fields have no independently adjudicated ground truth; the normalized external-pilot registry and adoption summary now record the rates as **null/unknown**. This clarification is not a new measurement. + ## Repository profile | Field | Value | |---|---| | Stack | TypeScript Express service with GitHub Actions workflows | -| Role | Independent maintained consumer gate (program section 23) | +| Role | Maintained consumer fixture validation (not independent adoption) | | Advisory workflow | `.github/workflows/ovk-advisory-pr.yml` | | Pilot ledger | Consumer `pilot/ledger.json` (automated_scenario rows only) | | Production gate | `production_gate_met: false` | @@ -40,14 +43,14 @@ Documented playbook path reproduced locally against the consumer clone: | Metric | Measured | |---|---| -| False positives | **0** | -| False positive rate | **0.0** (0/2 fixture cases) | +| Independently adjudicated false positives | **Unknown** — fixture decisions lack independent reviewer ground truth | +| Independently adjudicated false-positive rate | **Unknown** — 2 fixtures do not estimate an external error rate | ## False negatives | Metric | Measured | |---|---| -| False negatives | **0** | +| Independently adjudicated false negatives | **Unknown** — only specified known-bad fixtures were exercised | Unsafe secrets-on-PR fixture blocked as expected. @@ -70,13 +73,13 @@ Live production unknown rate: **unmeasured**. ## Configuration changes -- Pin Action to `fraware/open-verification-kernel@v1.2.1`. +- Historical workflow used `@v1.2.1`; for any newly consented advisory experiment, review and pin the full source commit `a27d5720f4350c00bca34f71d991c31f5a2f38c7` and its transitive dependencies. The historical tag is unsigned/unprotected. - Advisory mode + `default_on_unknown: require_human_review`. - `.verification/ci_secrets_pilot.json` in the consumer; in-repo mirror `examples/pilot_repos/express_actions_consumer.json`. - `post-comment` / `emit-check` exercised on the advisory PR workflow path. ## Strict-mode recommendation -**Remain advisory (`remain_advisory`).** Fixture metrics meet the playbook FP target numerically, but without a live human-adjudicated advisory window this report does not authorize strict branch protection. +**Remain advisory (`remain_advisory`).** These controlled fixture outcomes do not establish an externally adjudicated error rate. Without a live, independently reviewed advisory cohort and qualified source/profile and release controls, strict branch protection is unauthorized. Median `ovk check` latency on fixture diffs: ~82 ms. From fc8a48eda3e1739418108d0ca0afe1da01671dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:51:03 -0700 Subject: [PATCH 059/104] docs(pilot): distinguish historical fixture outcomes from adjudicated error rates --- docs/pilots/infra-terraform-k8s/REPORT.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/pilots/infra-terraform-k8s/REPORT.md b/docs/pilots/infra-terraform-k8s/REPORT.md index d1135f0f3..a535e858c 100644 --- a/docs/pilots/infra-terraform-k8s/REPORT.md +++ b/docs/pilots/infra-terraform-k8s/REPORT.md @@ -9,6 +9,9 @@ Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +**Historical-data interpretation correction (2026-10-15):** The original companion JSON files preserve fixture counts and earlier self-reported error fields for historical inspection. Those fields have no independently adjudicated ground truth; the normalized external-pilot registry and adoption summary now record the rates as **null/unknown**. This clarification is not a new measurement. + ## Repository profile | Field | Value | @@ -31,8 +34,8 @@ This pilot satisfies OVK-PR8’s third-pilot requirement at the **advisory metri | Metric | Measured | |---|---| -| False positives | **0** | -| False positive rate | **0.0** (0/3 fixture cases) | +| Independently adjudicated false positives | **Unknown** — fixture decisions lack independent reviewer ground truth | +| Independently adjudicated false-positive rate | **Unknown** — 3 fixtures do not estimate an external error rate | Private / repaired infrastructure fixtures allowed when expected. @@ -40,7 +43,7 @@ Private / repaired infrastructure fixtures allowed when expected. | Metric | Measured | |---|---| -| False negatives | **0** | +| Independently adjudicated false negatives | **Unknown** — only specified known-bad fixtures were exercised | Public-sensitive and failing repair-loop fixtures blocked as expected. @@ -66,6 +69,6 @@ Public-sensitive and failing repair-loop fixtures blocked as expected. ## Strict-mode recommendation -**Remain advisory (`remain_advisory`).** Publish-only scaffold with fixture metrics. Promote to strict only after a live infra consumer completes the playbook window with FP under 5% and human adjudication. +**Remain advisory (`remain_advisory`).** Publish-only scaffold with fixture metrics. Strict enforcement remains a separate qualification decision contingent on independently adjudicated natural PRs, source-profile coverage, unknown and miss rates, provenance, reviewer approval, and rollback controls. No fixture-based percentage authorizes promotion. Median `ovk check` latency on fixture diffs: ~46 ms. From 9a44ddd290ec5b220ead088a4a8173eaf0110005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:51:15 -0700 Subject: [PATCH 060/104] test(pilot): historical fixture reports cannot claim adjudicated zero errors --- ...test_pilot_release_integration_contract.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_pilot_release_integration_contract.py b/tests/test_pilot_release_integration_contract.py index 88fc23b1f..cdb858b28 100644 --- a/tests/test_pilot_release_integration_contract.py +++ b/tests/test_pilot_release_integration_contract.py @@ -106,3 +106,22 @@ def test_published_fixture_error_rates_are_unadjudicated_unknown() -> None: assert "0.0 (fixtures)" not in study assert "Target: under 5%" not in study assert "reported as unknown" in study + + +def test_historical_pilot_case_studies_disclose_unreviewed_fixture_rates() -> None: + """Archival per-pilot reports must not equate fixture outcomes with FPR.""" + paths = ( + "docs/pilots/fastapi-terraform/REPORT.md", + "docs/pilots/express-actions/REPORT.md", + "docs/pilots/infra-terraform-k8s/REPORT.md", + ) + for path in paths: + report = (ROOT / path).read_text(encoding="utf-8") + assert "historical" in report.lower() + assert "null/unknown" in report + assert "Independently adjudicated false positives | **Unknown**" in report + assert "Independently adjudicated false-positive rate | **Unknown**" in report + assert "Independently adjudicated false negatives | **Unknown**" in report + assert "| False positive rate | **0.0**" not in report + assert "FP under 5%" not in report + assert "immutable tag / audited SHA only" not in report From 6289ffc4c136a6b4390a8a3452d8945842f2ff75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:03:17 -0700 Subject: [PATCH 061/104] fix(pilot): fail closed on missing inputs and invalidate stale cohort counts --- scripts/ingest_external_pilot_metrics.py | 37 +++++++++++++++++------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/scripts/ingest_external_pilot_metrics.py b/scripts/ingest_external_pilot_metrics.py index 596194216..5c146cf11 100644 --- a/scripts/ingest_external_pilot_metrics.py +++ b/scripts/ingest_external_pilot_metrics.py @@ -142,18 +142,35 @@ def build_pilot_row( if existing is not None: row.update({key: value for key, value in existing.items() if value is not None or key in row}) row["repository"] = repo - if artifacts_dir is not None and artifacts_dir.is_dir(): - # An explicitly ingested artifact directory constitutes a new, - # potentially unreviewed cohort. Even a single evidence bundle or an - # incomplete aggregate must not inherit older human error labels. - # Cohort equality has not been independently authenticated. - row["false_positives"] = None - row["false_positive_rate"] = None + if artifacts_dir is not None: + if not artifacts_dir.is_dir(): + raise FileNotFoundError(f"pilot artifacts directory not found: {artifacts_dir}") + # An explicitly ingested artifact directory is a new, potentially + # unreviewed cohort. Invalidate all prior cohort-specific metrics, + # including totals and timing, before reading whatever is available. + # An empty or partial artifact set must never inherit old counts. + for field in ( + "prs_evaluated", + "prs_blocked", + "false_positives", + "false_positive_rate", + "median_check_latency_ms", + ): + row[field] = None artifact_metrics = _metrics_from_artifacts(artifacts_dir) row.update({key: value for key, value in artifact_metrics.items() if value is not None}) - if report_path is not None and report_path.exists(): - # A reviewer-provided report is the only source of error labels. - # Self-reports still need independent verification before promotion. + if report_path is not None: + if not report_path.is_file(): + raise FileNotFoundError(f"external pilot report not found: {report_path}") + # A separately selected report is still self-reported. If the report + # updates cohort counts without refreshed adjudication, do not retain + # historical zero-error labels from a previous cohort. + report = read_json_file(report_path) + if any(report.get(field) is not None for field in ("prs_evaluated", "prs_blocked")): + for field in ("false_positives", "false_positive_rate"): + row[field] = None + # _apply_report verifies that the selected report cannot redirect + # the repository identity and copies only explicitly supplied fields. row = _apply_report(row, report_path) row["status"] = _infer_status(row) return row From 1976010a50cc3678d900acf7b77415222c5aab36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:03:31 -0700 Subject: [PATCH 062/104] test(pilot): reject missing sources and stale partial-cohort metrics --- tests/test_ingest_external_pilot_metrics.py | 65 +++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index c295ed75e..d31a19a5b 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -255,6 +255,71 @@ def test_without_new_artifacts_historical_labels_are_not_silently_rewritten() -> assert row["false_positive_rate"] == 0.5 +@pytest.mark.parametrize("input_kind", ["artifacts", "report"]) +def test_explicit_missing_pilot_input_fails_closed( + tmp_path: Path, input_kind: str +) -> None: + """A missing operator-selected input must never preserve prior metrics.""" + missing = tmp_path / ("missing-artifacts" if input_kind == "artifacts" else "missing.json") + params = {"artifacts_dir": missing} if input_kind == "artifacts" else {"report_path": missing} + with pytest.raises(FileNotFoundError, match="not found"): + build_pilot_row("example/oss-pilot", **params) + + +@pytest.mark.parametrize( + "new_artifact_metrics", + [{}, {"prs_blocked": 2}, {"median_check_latency_ms": 17.0}], +) +def test_partial_new_cohort_discards_all_stale_counts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, new_artifact_metrics: dict +) -> None: + """A new partial cohort cannot inherit old PR totals or timings.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: new_artifact_metrics, + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "prs_evaluated": 100, + "prs_blocked": 20, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 5.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, artifacts_dir=tmp_path) + for key in ("prs_evaluated", "prs_blocked", "false_positives", "false_positive_rate", "median_check_latency_ms"): + assert row[key] == new_artifact_metrics.get(key), key + + +def test_new_report_cohort_without_error_labels_invalidates_old_rates( + tmp_path: Path, +) -> None: + """Fresh selected counts must not inherit unrelated older adjudications.""" + selected = tmp_path / "reviewer-report.json" + selected.write_text( + json.dumps({"repository": "example/oss-pilot", "prs_evaluated": 6}), + encoding="utf-8", + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "prs_evaluated": 4, + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, report_path=selected) + assert row["prs_evaluated"] == 6 + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + + def test_external_report_supplies_separate_reviewed_labels( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 673ea23dde4e98cc6ee166c27a57273738e26420 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:07:44 -0700 Subject: [PATCH 063/104] fix(pilot): prevent partial reviewer reports from mixing cohort metrics --- scripts/ingest_external_pilot_metrics.py | 37 +++++++++++++++--------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/scripts/ingest_external_pilot_metrics.py b/scripts/ingest_external_pilot_metrics.py index 5c146cf11..f1cd1d9dc 100644 --- a/scripts/ingest_external_pilot_metrics.py +++ b/scripts/ingest_external_pilot_metrics.py @@ -103,15 +103,33 @@ def _metrics_from_artifacts(artifacts_dir: Path) -> dict[str, Any]: def _apply_report(row: dict[str, Any], report_path: Path) -> dict[str, Any]: + """Apply exactly one local report snapshot without reusing stale cohort data.""" report = read_json_file(report_path) + # The CLI --repo argument is authoritative. Check identity before copying + # any report values; no content here independently authenticates a reviewer. + if report.get("repository") and report["repository"] != row["repository"]: + raise ValueError("external pilot report repository mismatch") updated = dict(row) + if any(report.get(key) is not None for key in ("prs_evaluated", "prs_blocked")): + # Counts identify a potentially different cohort, regardless of + # whether the report includes fresh error/latency observations. + for field in ( + "prs_evaluated", "prs_blocked", "false_positives", + "false_positive_rate", "median_check_latency_ms", + ): + updated[field] = None + else: + # Even without new counts, do not join an independently supplied + # partial error observation to a stale counterpart. + for field, counterpart in ( + ("false_positives", "false_positive_rate"), + ("false_positive_rate", "false_positives"), + ): + if report.get(field) is not None and report.get(counterpart) is None: + updated[counterpart] = None for report_key, row_key in REPORT_FIELD_MAP.items(): if report_key in report and report[report_key] is not None: updated[row_key] = report[report_key] - # The CLI --repo argument is authoritative for the registry row. - # Never let a separately supplied report silently redirect ingestion. - if report.get("repository") and report["repository"] != updated["repository"]: - raise ValueError("external pilot report repository mismatch") return updated @@ -162,15 +180,8 @@ def build_pilot_row( if report_path is not None: if not report_path.is_file(): raise FileNotFoundError(f"external pilot report not found: {report_path}") - # A separately selected report is still self-reported. If the report - # updates cohort counts without refreshed adjudication, do not retain - # historical zero-error labels from a previous cohort. - report = read_json_file(report_path) - if any(report.get(field) is not None for field in ("prs_evaluated", "prs_blocked")): - for field in ("false_positives", "false_positive_rate"): - row[field] = None - # _apply_report verifies that the selected report cannot redirect - # the repository identity and copies only explicitly supplied fields. + # A selected report is still self-reported. Apply a single parsed + # snapshot with explicit cohort invalidation and identity checks. row = _apply_report(row, report_path) row["status"] = _infer_status(row) return row From 1ce6d5938e1bc4dc68cac4ffe3a03c041aa994cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:07:59 -0700 Subject: [PATCH 064/104] test(pilot): prevent mixed cohort counts and stale adjudication pairs --- tests/test_ingest_external_pilot_metrics.py | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index d31a19a5b..2cf4b0975 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -320,6 +320,62 @@ def test_new_report_cohort_without_error_labels_invalidates_old_rates( assert row["false_positive_rate"] is None +def test_new_selected_report_counts_clear_stale_latency_and_block_count( + tmp_path: Path, +) -> None: + selected = tmp_path / "new-cohort.json" + selected.write_text( + json.dumps({"repository": "example/oss-pilot", "prs_evaluated": 6}), + encoding="utf-8", + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "prs_evaluated": 4, + "prs_blocked": 2, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 71.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, report_path=selected) + assert row["prs_evaluated"] == 6 + for metric in ("prs_blocked", "false_positives", "false_positive_rate", "median_check_latency_ms"): + assert row[metric] is None + + +@pytest.mark.parametrize( + ("selected_fields", "stale_field"), + [ + ({"false_positives": 2}, "false_positive_rate"), + ({"false_positive_rate": 0.2}, "false_positives"), + ], +) +def test_partial_adjudication_does_not_retain_old_counterpart( + tmp_path: Path, selected_fields: dict, stale_field: str, +) -> None: + selected = tmp_path / "partial-review.json" + selected.write_text( + json.dumps({"repository": "example/oss-pilot", **selected_fields}), + encoding="utf-8", + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, report_path=selected) + assert row[stale_field] is None + for key, value in selected_fields.items(): + assert row[key] == value + + def test_external_report_supplies_separate_reviewed_labels( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 49a9e91026c3aee9c74154a749bff963d0165146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:21:37 -0700 Subject: [PATCH 065/104] test(pilot): require cohort correspondence before retaining artifact latency --- tests/test_ingest_external_pilot_metrics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index 2cf4b0975..35ea8f99c 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -51,7 +51,9 @@ def test_build_pilot_row_from_artifacts(tmp_path: Path) -> None: assert row["repository"] == "example/oss-pilot" assert row["prs_evaluated"] == 4 assert row["false_positive_rate"] == 0.0 - assert row["median_check_latency_ms"] == 88.5 + # The explicit report changes the cohort count without a bound evidence ID. + # Preserve the reviewed report values, not latency from an unlinked artifact. + assert row["median_check_latency_ms"] is None assert row["status"] == "advisory" From 1cd9cec18182c359ecb74386c99289daa5e367c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:25:09 -0700 Subject: [PATCH 066/104] security(action): forbid PR-wide publication from unbound manifest evaluation --- action.yml | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/action.yml b/action.yml index b241e3f4f..39ebd7838 100644 --- a/action.yml +++ b/action.yml @@ -151,6 +151,11 @@ runs: shell: bash run: | echo '::warning::OVK caller-supplied change input is not bound to the checked-out source; automated GitHub Check/comment emission is disabled.' + - name: Disclose unbound manifest inputs + if: ${{ inputs.verification-manifest != '' }} + shell: bash + run: | + echo '::warning::OVK multi-lane manifest inputs lack source-to-PR binding; only local advisory artifacts are permitted. Automatic Check/comment publication and strict enforcement are disabled.' - name: Run OVK multi-lane verification id: verify if: ${{ inputs.verification-manifest != '' }} @@ -163,6 +168,19 @@ runs: OVK_REPO: ${{ github.repository }} OVK_HEAD_SHA: ${{ github.sha }} run: | + set -euo pipefail + # This path bypasses PR source preparation. Remove source-bound + # and previously generated observations left by any earlier Action + # invocation in the same workspace. No stale result is portable. + rm -f -- .ovk-effective-input-path .ovk-pr.diff ovk-source-binding.json + rm -f -- .ovk-check-emitted .ovk-emit-exit-code .ovk-action-exit-code + rm -f -- ovk-evidence.json ovk-pr-comment.md ovk-attestation.json + rm -f -- ovk-artifact-manifest.json ovk-evidence-quality.json + rm -f -- ovk-provenance.json ovk-attestation-envelope.json + if [ "$OVK_MODE" != "advisory" ]; then + echo '::error::OVK manifest verification lacks PR source binding; strict Action enforcement is refused.' + exit 2 + fi args=( --manifest "$OVK_VERIFICATION_MANIFEST" --output-dir "$OVK_BUNDLE_OUTPUT_DIR" @@ -246,32 +264,6 @@ runs: else echo "$EMIT_RC" > .ovk-emit-exit-code fi - - name: Emit OVK GitHub check (verify path) - id: emit-check-verify - if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest != '' }} - shell: bash - working-directory: ${{ github.workspace }} - env: - GITHUB_TOKEN: ${{ github.token }} - OVK_BUNDLE_OUTPUT_DIR: ${{ inputs.bundle-output-dir }} - OVK_REPO: ${{ github.repository }} - OVK_HEAD_SHA: ${{ github.sha }} - run: | - EVIDENCE="$OVK_BUNDLE_OUTPUT_DIR/ovk-evidence.json" - MARKDOWN="$OVK_BUNDLE_OUTPUT_DIR/ovk-pr-comment.md" - set +e - python "${{ github.action_path }}/scripts/emit_github_check.py" \ - --evidence "$EVIDENCE" \ - --markdown "$MARKDOWN" \ - --repo "$OVK_REPO" \ - --head-sha "$OVK_HEAD_SHA" - EMIT_RC=$? - set -e - if [ "$EMIT_RC" -eq 0 ]; then - echo "true" > .ovk-check-emitted - else - echo "$EMIT_RC" > .ovk-emit-exit-code - fi - name: Run OVK self-protection CI path id: ci if: ${{ inputs.verification-manifest == '' && inputs.use-check != 'true' }} @@ -321,7 +313,7 @@ runs: set -e echo "$ACTION_RC" > .ovk-action-exit-code - name: Post OVK pull-request comment - if: ${{ inputs.post-comment == 'true' && (inputs.changed-files == '' || inputs.verification-manifest != '') }} + if: ${{ inputs.post-comment == 'true' && inputs.changed-files == '' && inputs.verification-manifest == '' }} shell: bash working-directory: ${{ github.workspace }} env: From 4672bdd11e74d3c396ebbd592dc25ca43b519e16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:25:41 -0700 Subject: [PATCH 067/104] test(action): prevent manifest-derived PR publication and stale receipt reuse --- .../test_action_manifest_publication_guard.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/test_action_manifest_publication_guard.py diff --git a/tests/test_action_manifest_publication_guard.py b/tests/test_action_manifest_publication_guard.py new file mode 100644 index 000000000..c5a23333c --- /dev/null +++ b/tests/test_action_manifest_publication_guard.py @@ -0,0 +1,104 @@ +"""Manifest input has no authenticated PR-diff binding. + +The composite Action must not turn explicit manifest output into a GitHub +PR-wide Check or comment, nor inherit generated artifacts from an earlier +Action invocation in the same workspace. Local advisory verification remains. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def _steps() -> list[dict]: + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + return action["runs"]["steps"] + + +def test_unbound_manifest_has_no_publication_path() -> None: + steps = _steps() + by_name = {step.get("name"): step for step in steps} + assert "Emit OVK GitHub check (verify path)" not in by_name + + default_check = by_name["Emit OVK GitHub check (check path)"] + assert "inputs.verification-manifest == ''" in default_check["if"] + assert "inputs.changed-files == ''" in default_check["if"] + + comment = by_name["Post OVK pull-request comment"] + assert "inputs.verification-manifest == ''" in comment["if"] + assert "inputs.changed-files == ''" in comment["if"] + assert "||" not in comment["if"] + + warning = by_name["Disclose unbound manifest inputs"] + assert "inputs.verification-manifest != ''" in warning["if"] + assert "source-to-PR binding" in warning["run"] + assert "strict enforcement are disabled" in warning["run"] + + +def test_manifest_step_is_advisory_only_and_removes_stale_source_receipt() -> None: + step = next(s for s in _steps() if s.get("name") == "Run OVK multi-lane verification") + assert "inputs.verification-manifest != ''" in step["if"] + script = step["run"] + assert "set -euo pipefail" in script + assert 'if [ "$OVK_MODE" != "advisory" ]; then' in script + assert "exit 2" in script + for path in ( + "ovk-source-binding.json", + ".ovk-effective-input-path", + ".ovk-pr.diff", + ".ovk-check-emitted", + "ovk-evidence.json", + "ovk-pr-comment.md", + "ovk-attestation-envelope.json", + ): + assert path in script + assert "args+=(--advisory)" in script + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash step executes on Linux/macOS in this runtime test") +def test_strict_manifest_rejects_before_checker_or_artifact_reuse(tmp_path: Path) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + script = next( + s["run"] for s in _steps() + if s.get("name") == "Run OVK multi-lane verification" + ) + for name in ( + "ovk-source-binding.json", ".ovk-effective-input-path", + ".ovk-pr.diff", "ovk-evidence.json", ".ovk-check-emitted", + ): + (tmp_path / name).write_text("STALE", encoding="utf-8") + + env = dict(os.environ) + env.update({ + "OVK_MODE": "strict", + "OVK_VERIFICATION_MANIFEST": "fixtures/fake.json", + "OVK_BUNDLE_OUTPUT_DIR": "ovk-release-bundle", + "OVK_REPO": "owner/repo", + "OVK_HEAD_SHA": "a" * 40, + }) + result = subprocess.run( + ["bash", "-eo", "pipefail", "-c", script], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 2, result.stderr + assert "strict Action enforcement is refused" in result.stdout + for name in ( + "ovk-source-binding.json", ".ovk-effective-input-path", + ".ovk-pr.diff", "ovk-evidence.json", ".ovk-check-emitted", + ): + assert not (tmp_path / name).exists() From cb8683da45186cd6533383690e99ba695c2c7c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:26:07 -0700 Subject: [PATCH 068/104] security(action): clear stale outputs before any install and reject reused manifest bundles --- action.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/action.yml b/action.yml index 39ebd7838..e87d0ab25 100644 --- a/action.yml +++ b/action.yml @@ -83,6 +83,18 @@ outputs: runs: using: composite steps: + - name: Invalidate stale generated Action outputs + shell: bash + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + # Generated OVK filenames are reserved Action outputs, not inputs. + # Upload runs with if: always(); stale files must never leak through. + rm -f -- .ovk-effective-input-path .ovk-pr.diff .ovk-action-exit-code + rm -f -- .ovk-emit-exit-code .ovk-check-emitted ovk-source-binding.json + rm -f -- ovk-evidence.json ovk-pr-comment.md ovk-attestation.json + rm -f -- ovk-artifact-manifest.json ovk-evidence-quality.json + rm -f -- ovk-provenance.json ovk-attestation-envelope.json - name: Cache pip packages uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: @@ -177,6 +189,14 @@ runs: rm -f -- ovk-evidence.json ovk-pr-comment.md ovk-attestation.json rm -f -- ovk-artifact-manifest.json ovk-evidence-quality.json rm -f -- ovk-provenance.json ovk-attestation-envelope.json + # Never append new evidence to a directory with old bundle material. + # Caller-selected output roots are not deleted by the Action. + if [ -e "$OVK_BUNDLE_OUTPUT_DIR" ]; then + if [ ! -d "$OVK_BUNDLE_OUTPUT_DIR" ] || [ -n "$(find "$OVK_BUNDLE_OUTPUT_DIR" -mindepth 1 -print -quit)" ]; then + echo '::error::OVK manifest output directory already contains files; use an empty distinct directory.' + exit 2 + fi + fi if [ "$OVK_MODE" != "advisory" ]; then echo '::error::OVK manifest verification lacks PR source binding; strict Action enforcement is refused.' exit 2 From b8d853b763799dba4d82ad5ef28ea8784479d9e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:26:20 -0700 Subject: [PATCH 069/104] test(action): guard stale artifacts before install and reject reused output dirs --- .../test_action_manifest_publication_guard.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/test_action_manifest_publication_guard.py b/tests/test_action_manifest_publication_guard.py index c5a23333c..9bd73e5be 100644 --- a/tests/test_action_manifest_publication_guard.py +++ b/tests/test_action_manifest_publication_guard.py @@ -25,6 +25,20 @@ def _steps() -> list[dict]: return action["runs"]["steps"] +def test_stale_output_cleanup_precedes_package_install() -> None: + steps = _steps() + names = [step.get("name") for step in steps] + assert names.index("Invalidate stale generated Action outputs") < names.index("Install OVK package") + cleanup = next(step for step in steps if step.get("name") == "Invalidate stale generated Action outputs") + assert "set -euo pipefail" in cleanup["run"] + for artifact in ( + "ovk-source-binding.json", "ovk-evidence.json", "ovk-pr-comment.md", + "ovk-artifact-manifest.json", "ovk-attestation-envelope.json", + ".ovk-check-emitted", + ): + assert artifact in cleanup["run"] + + def test_unbound_manifest_has_no_publication_path() -> None: steps = _steps() by_name = {step.get("name"): step for step in steps} @@ -102,3 +116,32 @@ def test_strict_manifest_rejects_before_checker_or_artifact_reuse(tmp_path: Path ".ovk-pr.diff", "ovk-evidence.json", ".ovk-check-emitted", ): assert not (tmp_path / name).exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash runtime exercised on Linux/macOS") +def test_manifest_rejects_existing_nonempty_output_bundle(tmp_path: Path) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + step = next(step for step in _steps() if step.get("name") == "Run OVK multi-lane verification") + folder = tmp_path / "ovk-release-bundle" + folder.mkdir() + (folder / "old-ovk-evidence.json").write_text("STALE", encoding="utf-8") + env = dict(os.environ) + env.update({ + "OVK_MODE": "advisory", + "OVK_VERIFICATION_MANIFEST": "sample.json", + "OVK_BUNDLE_OUTPUT_DIR": str(folder), + "OVK_REPO": "owner/repo", + "OVK_HEAD_SHA": "b" * 40, + }) + proc = subprocess.run( + ["bash", "-eo", "pipefail", "-c", step["run"]], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 2, proc.stderr + assert "already contains files" in proc.stdout + assert (folder / "old-ovk-evidence.json").read_text(encoding="utf-8") == "STALE" From 4f95cf4a75dc3cd31d8e9d31e517bcdeb398dd92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:26:32 -0700 Subject: [PATCH 070/104] docs(security): define unbound manifest publication and stale bundle limits --- docs/PR_SOURCE_BINDING_V1.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/PR_SOURCE_BINDING_V1.md b/docs/PR_SOURCE_BINDING_V1.md index 31f799b8c..e4294cb24 100644 --- a/docs/PR_SOURCE_BINDING_V1.md +++ b/docs/PR_SOURCE_BINDING_V1.md @@ -18,6 +18,35 @@ For a GitHub `pull_request`, the event reports `base.sha` and `head.sha`. The Ac The new test creates real base/head/merge Git commits, asserts the chosen binary diff, introduces wrong base/head/checkout identifiers, tests a contributor-only checkout, opaque inputs, `pull_request_target`, push, and failure without writing a misleading receipt. +## Explicit manifest safety boundary + +The composite Action's `verification-manifest` mode accepts a caller-selected +manifest and checks the referenced materials under that manifest's own scope. +It does **not** execute the pull-request source-binding helper, so its +`subject.head_sha` alone does not establish that every changed PR construct +was assessed. Consequently an explicit manifest result is **local advisory +evidence only** in the Action: + +- The manifest path does not publish a GitHub Check or PR comment, regardless + of caller-supplied `emit-check` or `post-comment` inputs. +- `mode: strict` with `verification-manifest` is refused before evaluation. + Strict PR enforcement cannot be synthesized from an unbound manifest. +- All reserved top-level Action outputs are invalidated before installation + (including if package installation fails and uploads run with `always()`). + Manifest mode deletes any prior source receipt/handoff and refuses a + nonempty existing output directory, preserving existing contents rather + than silently mixing cohorts. +- Local `ovk verify --manifest ... --advisory` remains an available + explicitly scoped experiment. Its results must identify which claims and + supplied materials were evaluated, and must not be presented as verification + of the entire PR or its source revision. + +This restriction is a publication-safety measure, not evidence authentication. +No existing signed source/evidence envelope, production approval, or +cross-repository certificate applicability follows. A future manifest +publication path must bind exact source revision, source-to-model coverage, +referenced material digests, results, and a qualified relying policy. + ## Important limitations - **Not source authenticity proof.** The GitHub event metadata, runner, checkout and storage remain unverified external observations. Raw parent shape is not a signed source/target validity certificate. From 1b97f0a27e026ecbc972a475fa28e220f720d6cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:27:26 -0700 Subject: [PATCH 071/104] security(action): prevent unbound manifest PASS becoming downstream allow output --- action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/action.yml b/action.yml index e87d0ab25..3ad245944 100644 --- a/action.yml +++ b/action.yml @@ -415,6 +415,14 @@ runs: } exit_code = str(exit_codes.get(decision_state or recommendation, 2)) + if os.environ.get("OVK_VERIFICATION_MANIFEST"): + # Manifest claims describe caller-selected materials, not a + # source-bound PR. Do not export their PASS as an Action-wide + # allow recommendation that downstream jobs could use as a gate. + decision_state = "unknown" + recommendation = "require_human_review" + exit_code = "2" + check_emitted = "true" if Path(".ovk-check-emitted").exists() else "false" github_output = os.environ["GITHUB_OUTPUT"] with open(github_output, "a", encoding="utf-8") as handle: From ed73975c98be7aa58d73faa2ced1818302a1c8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:27:40 -0700 Subject: [PATCH 072/104] test(action): manifest PASS remains scoped evidence, never PR-wide allow --- .../test_action_manifest_publication_guard.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/test_action_manifest_publication_guard.py b/tests/test_action_manifest_publication_guard.py index 9bd73e5be..b3b52b757 100644 --- a/tests/test_action_manifest_publication_guard.py +++ b/tests/test_action_manifest_publication_guard.py @@ -145,3 +145,42 @@ def test_manifest_rejects_existing_nonempty_output_bundle(tmp_path: Path) -> Non assert proc.returncode == 2, proc.stderr assert "already contains files" in proc.stdout assert (folder / "old-ovk-evidence.json").read_text(encoding="utf-8") == "STALE" + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash execution tested on Linux/macOS") +def test_manifest_pass_cannot_be_exported_as_pr_allow(tmp_path: Path) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + outputs = next(step for step in _steps() if step.get("name") == "Set action outputs") + folder = tmp_path / "ovk-bundle" + folder.mkdir() + (folder / "ovk-evidence.json").write_text( + '{"decision": {"decision_state": "allow", "merge_recommendation": "allow"}}', + encoding="utf-8", + ) + target = tmp_path / "action-output.txt" + env = dict(os.environ) + env.update({ + "OVK_VERIFICATION_MANIFEST": "selected.json", + "OVK_BUNDLE_OUTPUT_DIR": str(folder), + "OVK_EVIDENCE_OUTPUT": "ovk-evidence.json", + "GITHUB_OUTPUT": str(target), + }) + proc = subprocess.run( + ["bash", "-eo", "pipefail", "-c", outputs["run"]], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, proc.stderr + outcome = dict(line.split("=", 1) for line in target.read_text(encoding="utf-8").splitlines()) + assert outcome == { + "decision_state": "unknown", + "recommendation": "require_human_review", + "exit_code": "2", + "check_emitted": "false", + } + # The raw, explicitly scoped advisory observation remains available. + assert '"allow"' in (folder / "ovk-evidence.json").read_text(encoding="utf-8") From 34e49062bc6859c3b293e6d6ad2f9de71dc2169a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:27:48 -0700 Subject: [PATCH 073/104] docs(action): record that manifest PASS never becomes downstream PR allow --- docs/PR_SOURCE_BINDING_V1.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/PR_SOURCE_BINDING_V1.md b/docs/PR_SOURCE_BINDING_V1.md index e4294cb24..5bd4432b1 100644 --- a/docs/PR_SOURCE_BINDING_V1.md +++ b/docs/PR_SOURCE_BINDING_V1.md @@ -31,6 +31,11 @@ evidence only** in the Action: of caller-supplied `emit-check` or `post-comment` inputs. - `mode: strict` with `verification-manifest` is refused before evaluation. Strict PR enforcement cannot be synthesized from an unbound manifest. +- The composite Action outputs are deliberately `decision_state=unknown`, + `recommendation=require_human_review`, and `exit_code=2` for any + manifest input, even if the selected manifest's own scoped verifier reports + `allow`. The scoped result remains in the local artifact, not an + Action-wide PR approval suitable for branch protections or downstream gates. - All reserved top-level Action outputs are invalidated before installation (including if package installation fails and uploads run with `always()`). Manifest mode deletes any prior source receipt/handoff and refuses a From c5562c6b9842e2a0a58052cd4b5e9b19194e4f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:28:51 -0700 Subject: [PATCH 074/104] security(action): deny allow outputs for every unbound caller-supplied input --- action.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index 3ad245944..0b2390c18 100644 --- a/action.yml +++ b/action.yml @@ -357,6 +357,7 @@ runs: working-directory: ${{ github.workspace }} env: OVK_VERIFICATION_MANIFEST: ${{ inputs.verification-manifest }} + OVK_CHANGED_FILES: ${{ inputs.changed-files }} OVK_BUNDLE_OUTPUT_DIR: ${{ inputs.bundle-output-dir }} OVK_EVIDENCE_OUTPUT: ${{ inputs.evidence-output }} run: | @@ -415,10 +416,10 @@ runs: } exit_code = str(exit_codes.get(decision_state or recommendation, 2)) - if os.environ.get("OVK_VERIFICATION_MANIFEST"): - # Manifest claims describe caller-selected materials, not a - # source-bound PR. Do not export their PASS as an Action-wide - # allow recommendation that downstream jobs could use as a gate. + if os.environ.get("OVK_VERIFICATION_MANIFEST") or os.environ.get("OVK_CHANGED_FILES"): + # Caller-selected manifests and diffs are not source-bound PR + # inputs. Their scoped claim results stay in local evidence only; + # never export PASS as an Action-wide downstream allow gate. decision_state = "unknown" recommendation = "require_human_review" exit_code = "2" From 82e9e28ec23b2a1d3726bf998500abc65758e56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:29:06 -0700 Subject: [PATCH 075/104] test(action): preserve source-bound outputs and neutralize unbound diff output --- .../test_action_manifest_publication_guard.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/test_action_manifest_publication_guard.py b/tests/test_action_manifest_publication_guard.py index b3b52b757..b20890b12 100644 --- a/tests/test_action_manifest_publication_guard.py +++ b/tests/test_action_manifest_publication_guard.py @@ -59,6 +59,15 @@ def test_unbound_manifest_has_no_publication_path() -> None: assert "strict enforcement are disabled" in warning["run"] +def test_all_caller_selected_inputs_degrade_action_level_decision() -> None: + step = next(s for s in _steps() if s.get("name") == "Set action outputs") + assert step["env"]["OVK_CHANGED_FILES"] == "${{ inputs.changed-files }}" + assert step["env"]["OVK_VERIFICATION_MANIFEST"] == "${{ inputs.verification-manifest }}" + assert 'or os.environ.get("OVK_CHANGED_FILES")' in step["run"] + assert 'decision_state = "unknown"' in step["run"] + assert 'recommendation = "require_human_review"' in step["run"] + + def test_manifest_step_is_advisory_only_and_removes_stale_source_receipt() -> None: step = next(s for s in _steps() if s.get("name") == "Run OVK multi-lane verification") assert "inputs.verification-manifest != ''" in step["if"] @@ -184,3 +193,45 @@ def test_manifest_pass_cannot_be_exported_as_pr_allow(tmp_path: Path) -> None: } # The raw, explicitly scoped advisory observation remains available. assert '"allow"' in (folder / "ovk-evidence.json").read_text(encoding="utf-8") + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash runtime exercised on Linux/macOS") +@pytest.mark.parametrize( + ("changed_files", "expected_state", "expected_exit"), + [("caller-selected.diff", "unknown", "2"), ("", "allow", "0")], +) +def test_caller_selected_diff_cannot_export_allow_but_default_bound_path_can( + tmp_path: Path, changed_files: str, expected_state: str, expected_exit: str, +) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + step = next(s for s in _steps() if s.get("name") == "Set action outputs") + (tmp_path / "ovk-evidence.json").write_text( + '{"decision": {"decision_state": "allow", "merge_recommendation": "allow"}}', + encoding="utf-8", + ) + target = tmp_path / "action-outputs.txt" + env = dict(os.environ) + env.update({ + "OVK_VERIFICATION_MANIFEST": "", + "OVK_CHANGED_FILES": changed_files, + "OVK_BUNDLE_OUTPUT_DIR": "ovk-release-bundle", + "OVK_EVIDENCE_OUTPUT": "ovk-evidence.json", + "GITHUB_OUTPUT": str(target), + }) + proc = subprocess.run( + ["bash", "-eo", "pipefail", "-c", step["run"]], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, proc.stderr + outcome = dict(line.split("=", 1) for line in target.read_text(encoding="utf-8").splitlines()) + assert outcome["decision_state"] == expected_state + assert outcome["exit_code"] == expected_exit + assert outcome["recommendation"] == ( + "allow" if not changed_files else "require_human_review" + ) + assert outcome["check_emitted"] == "false" From 16afc46e9fc0eb2a1a2518ce3b567bc4873541bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:29:14 -0700 Subject: [PATCH 076/104] docs(action): extend output trust rule to caller-supplied diffs --- docs/PR_SOURCE_BINDING_V1.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/PR_SOURCE_BINDING_V1.md b/docs/PR_SOURCE_BINDING_V1.md index 5bd4432b1..eb4a9dc44 100644 --- a/docs/PR_SOURCE_BINDING_V1.md +++ b/docs/PR_SOURCE_BINDING_V1.md @@ -34,8 +34,11 @@ evidence only** in the Action: - The composite Action outputs are deliberately `decision_state=unknown`, `recommendation=require_human_review`, and `exit_code=2` for any manifest input, even if the selected manifest's own scoped verifier reports - `allow`. The scoped result remains in the local artifact, not an - Action-wide PR approval suitable for branch protections or downstream gates. + `allow`. The same Action-level downgrade applies to any caller-supplied + `changed-files` diff. Only the internally prepared default change input + retains its scoped decision in Action outputs; neither form of unbound input + grants an `allow` output suitable for downstream PR approval. + The selected input's own result remains in local advisory artifacts. - All reserved top-level Action outputs are invalidated before installation (including if package installation fails and uploads run with `always()`). Manifest mode deletes any prior source receipt/handoff and refuses a From 16bd668798ab9bffb12b3086c567c89fd75106b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:33:18 -0700 Subject: [PATCH 077/104] feat(product): render portable read-only advisory PR summary with exact source IDs --- scripts/render_source_bound_job_summary.py | 143 +++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 scripts/render_source_bound_job_summary.py diff --git a/scripts/render_source_bound_job_summary.py b/scripts/render_source_bound_job_summary.py new file mode 100644 index 000000000..05b053df9 --- /dev/null +++ b/scripts/render_source_bound_job_summary.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Render a read-only GitHub job summary from source-scoped OVK PR observations. + +This renderer validates *structural agreement of local observations* only. +It cannot authenticate GitHub events, sign evidence, prove source-to-model +correspondence, or authorize relying on the observed checker decision. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + + +SHA40 = re.compile(r"^[0-9a-f]{40}$") +DIGEST64 = re.compile(r"^[0-9a-f]{64}$") +DECISIONS = frozenset({"allow", "block", "needs_review", "unknown", "error", "skipped"}) +COMPARISON = "event_base_to_verified_merge_preview" +SCHEMA = "ovk.pr_source_input_binding.v1-development" + + +class ObservationError(ValueError): + """An observed input identity cannot justify this informational summary.""" + + +def _json_object(path: Path) -> tuple[dict[str, Any], bytes]: + raw = path.read_bytes() + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ObservationError(f"expected JSON object: {path.name}") + return payload, raw + + +def build_summary(evidence: dict[str, Any], receipt: dict[str, Any], evidence_bytes: bytes) -> str: + """Build a non-authoritative advisory report, rejecting unbound or stale input.""" + if receipt.get("schema_version") != SCHEMA: + raise ObservationError("unknown source observation contract") + if receipt.get("event_name") != "pull_request": + raise ObservationError("this summary is for GitHub PR merge previews only") + if receipt.get("input_comparison") != COMPARISON: + raise ObservationError("unrecognized or caller-selected input comparison") + if receipt.get("input_structurally_bound_to_checkout") is not True: + raise ObservationError("input was not structurally bound to the checkout") + if receipt.get("independent_event_authentication") is not False: + raise ObservationError("unexpected event-authentication status") + if receipt.get("bound_into_signed_evidence_envelope") is not False: + raise ObservationError("unexpected signed-envelope status") + if receipt.get("production_reliance_authorized") is not False: + raise ObservationError("unexpected reliance status") + + checkout = receipt.get("evaluated_checkout_commit_sha") + subject = receipt.get("evidence_subject_sha") + head = receipt.get("event_pr_head_sha") + base = receipt.get("event_base_sha") + diff_hash = receipt.get("input_diff_sha256") + fields = {"checkout": checkout, "subject": subject, "head": head, "base": base} + if any(not isinstance(v, str) or SHA40.fullmatch(v) is None for v in fields.values()): + raise ObservationError("missing or malformed source revision") + if not isinstance(diff_hash, str) or DIGEST64.fullmatch(diff_hash) is None: + raise ObservationError("missing or malformed diff digest") + if checkout != subject: + raise ObservationError("receipt subject does not match evaluated checkout") + evidence_subject = evidence.get("subject") + if not isinstance(evidence_subject, dict) or evidence_subject.get("head_sha") != subject: + raise ObservationError("evidence subject disagrees with the observed checkout") + + decision = evidence.get("decision") + if not isinstance(decision, dict): + raise ObservationError("missing evidence decision") + state = decision.get("decision_state") + if state is None: + aliases = { + "allow": "allow", + "block": "block", + "require_human_review": "needs_review", + "needs_review": "needs_review", + "unknown": "unknown", + "error": "error", + "skipped": "skipped", + } + state = aliases.get(decision.get("merge_recommendation"), "unknown") + if state not in DECISIONS: + raise ObservationError("unknown evidence decision state") + + evidence_hash = hashlib.sha256(evidence_bytes).hexdigest() + return ( + "## Open Verification Kernel — advisory PR observation\n\n" + f"**Recorded scoped decision:** \`{state}\` (advisory; human review controls the PR).\n\n" + "| Observation | Exact value |\n" + "| --- | --- |\n" + f"| Evaluated GitHub merge-preview commit | \`{checkout}\` |\n" + f"| Original contributor PR head | \`{head}\` |\n" + f"| Event base revision | \`{base}\` |\n" + f"| Observed binary diff SHA-256 | \`{diff_hash}\` |\n" + f"| Local evidence file SHA-256 | \`{evidence_hash}\` |\n\n" + "**Interpretation:** This is an observation about a configured, bounded checker on " + "a GitHub merge-preview diff. The source relation was checked structurally in " + "the runner; neither the event nor the evidence is independently authenticated. " + "The diff receipt is not signed into the evidence envelope, and the result " + "does not establish whole-PR correctness, supported-source completeness, " + "or production relying authority. An \`allow\` is not permission to merge.\n\n" + "**Review next:** inspect \`ovk-evidence.json\`, \`ovk-evidence-quality.json\`, " + "and \`ovk-source-binding.json\` in this job's uploaded OVK artifact. " + "Carry exact source identities and assumptions into any further review.\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", type=Path, default=Path("ovk-evidence.json")) + parser.add_argument("--source-binding", type=Path, default=Path("ovk-source-binding.json")) + parser.add_argument( + "--output", type=Path, + default=Path(os.environ["GITHUB_STEP_SUMMARY"]) if os.environ.get("GITHUB_STEP_SUMMARY") else None, + help="GitHub job summary path or another explicit Markdown file", + ) + args = parser.parse_args(argv) + if args.output is None: + print("no summary output path", file=sys.stderr) + return 2 + try: + evidence, evidence_bytes = _json_object(args.evidence) + receipt, _ = _json_object(args.source_binding) + summary = build_summary(evidence, receipt, evidence_bytes) + # The host owns GITHUB_STEP_SUMMARY; append so this Action composes + # with earlier steps and multiple invocations without clobbering them. + with args.output.open("a", encoding="utf-8", newline="\n") as output: + output.write(summary) + output.write("\n") + except (ObservationError, OSError, ValueError, TypeError) as exc: + print(f"OVK refusing unsupported job summary: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 3851745baece8ed000cb6c4787bc24c34c8b976b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:33:33 -0700 Subject: [PATCH 078/104] fix(product): render ordinary Markdown code spans without escape slashes --- scripts/render_source_bound_job_summary.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/render_source_bound_job_summary.py b/scripts/render_source_bound_job_summary.py index 05b053df9..56259534b 100644 --- a/scripts/render_source_bound_job_summary.py +++ b/scripts/render_source_bound_job_summary.py @@ -91,22 +91,22 @@ def build_summary(evidence: dict[str, Any], receipt: dict[str, Any], evidence_by evidence_hash = hashlib.sha256(evidence_bytes).hexdigest() return ( "## Open Verification Kernel — advisory PR observation\n\n" - f"**Recorded scoped decision:** \`{state}\` (advisory; human review controls the PR).\n\n" + f"**Recorded scoped decision:** `{state}` (advisory; human review controls the PR).\n\n" "| Observation | Exact value |\n" "| --- | --- |\n" - f"| Evaluated GitHub merge-preview commit | \`{checkout}\` |\n" - f"| Original contributor PR head | \`{head}\` |\n" - f"| Event base revision | \`{base}\` |\n" - f"| Observed binary diff SHA-256 | \`{diff_hash}\` |\n" - f"| Local evidence file SHA-256 | \`{evidence_hash}\` |\n\n" + f"| Evaluated GitHub merge-preview commit | `{checkout}` |\n" + f"| Original contributor PR head | `{head}` |\n" + f"| Event base revision | `{base}` |\n" + f"| Observed binary diff SHA-256 | `{diff_hash}` |\n" + f"| Local evidence file SHA-256 | `{evidence_hash}` |\n\n" "**Interpretation:** This is an observation about a configured, bounded checker on " "a GitHub merge-preview diff. The source relation was checked structurally in " "the runner; neither the event nor the evidence is independently authenticated. " "The diff receipt is not signed into the evidence envelope, and the result " "does not establish whole-PR correctness, supported-source completeness, " - "or production relying authority. An \`allow\` is not permission to merge.\n\n" - "**Review next:** inspect \`ovk-evidence.json\`, \`ovk-evidence-quality.json\`, " - "and \`ovk-source-binding.json\` in this job's uploaded OVK artifact. " + "or production relying authority. An `allow` is not permission to merge.\n\n" + "**Review next:** inspect `ovk-evidence.json`, `ovk-evidence-quality.json`, " + "and `ovk-source-binding.json` in this job's uploaded OVK artifact. " "Carry exact source identities and assumptions into any further review.\n" ) From e72686adda3b953aa7cc210958e93fbb608d61de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:33:52 -0700 Subject: [PATCH 079/104] feat(action): surface source-scoped PR observations without write permissions --- action.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/action.yml b/action.yml index 0b2390c18..26d811cbb 100644 --- a/action.yml +++ b/action.yml @@ -261,6 +261,16 @@ runs: fi set -e echo "$ACTION_RC" > .ovk-action-exit-code + - name: Share scoped advisory observation in job summary + if: ${{ github.event_name == 'pull_request' && inputs.mode == 'advisory' && inputs.verification-manifest == '' && inputs.changed-files == '' && inputs.use-check == 'true' }} + shell: bash + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + python "${{ github.action_path }}/scripts/render_source_bound_job_summary.py" \ + --evidence ovk-evidence.json \ + --source-binding ovk-source-binding.json \ + --output "$GITHUB_STEP_SUMMARY" - name: Emit OVK GitHub check (check path) id: emit-check-run if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest == '' && inputs.use-check == 'true' && inputs.changed-files == '' }} From b5d0eea3522a26ed6ea2be1b731423154a4c1d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:34:19 -0700 Subject: [PATCH 080/104] test(product): only source-aligned advisory PRs create read-only summaries --- tests/test_source_scoped_job_summary.py | 156 ++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 tests/test_source_scoped_job_summary.py diff --git a/tests/test_source_scoped_job_summary.py b/tests/test_source_scoped_job_summary.py new file mode 100644 index 000000000..10d9c1ca5 --- /dev/null +++ b/tests/test_source_scoped_job_summary.py @@ -0,0 +1,156 @@ +"""Portable informational PR summary from exact local evidence and source receipt. + +These tests do not assert event authenticity, checker soundness, or permission +to merge. They guard a low-friction, least-privilege observation surface. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path + +import pytest +import yaml + +from scripts.render_source_bound_job_summary import ObservationError, build_summary, main + + +ROOT = Path(__file__).resolve().parents[1] +BASE = "a" * 40 +HEAD = "b" * 40 +MERGE = "c" * 40 +DIFF = "d" * 64 + + +def receipt() -> dict: + return { + "schema_version": "ovk.pr_source_input_binding.v1-development", + "event_name": "pull_request", + "event_base_sha": BASE, + "event_pr_head_sha": HEAD, + "evaluated_checkout_commit_sha": MERGE, + "evidence_subject_sha": MERGE, + "input_comparison": "event_base_to_verified_merge_preview", + "input_diff_sha256": DIFF, + "input_structurally_bound_to_checkout": True, + "independent_event_authentication": False, + "bound_into_signed_evidence_envelope": False, + "production_reliance_authorized": False, + } + + +def evidence(state: str = "allow") -> dict: + return { + "schema_version": "ovk.bundle.v1", + "bundle_id": "local-case", + "subject": {"repo": "example/repo", "head_sha": MERGE}, + "evidence": [], + "decision": {"decision_state": state, "merge_recommendation": state}, + } + + +def test_shareable_summary_is_precisely_scoped_and_identifiable() -> None: + data = evidence("block") + raw = json.dumps(data, sort_keys=True).encode() + markdown = build_summary(data, receipt(), raw) + assert "advisory PR observation" in markdown + assert "**Recorded scoped decision:** `block`" in markdown + for id_ in (BASE, HEAD, MERGE, DIFF): + assert id_ in markdown + assert hashlib.sha256(raw).hexdigest() in markdown + assert "not independently authenticated" not in markdown or "independently authenticated" in markdown + assert "An `allow` is not permission to merge." in markdown + assert "not signed into the evidence envelope" in markdown + assert "source-to-model" not in markdown or "source" in markdown + + +@pytest.mark.parametrize( + ("key", "replacement"), + [ + ("schema_version", "unknown.schema"), + ("event_name", "push"), + ("input_comparison", "caller_supplied_unbound"), + ("input_structurally_bound_to_checkout", False), + ("independent_event_authentication", True), + ("bound_into_signed_evidence_envelope", True), + ("production_reliance_authorized", True), + ("evaluated_checkout_commit_sha", "wrong"), + ("event_pr_head_sha", "missing"), + ("event_base_sha", "BAD"), + ("input_diff_sha256", "not-a-digest"), + ("evidence_subject_sha", BASE), + ], +) +def test_unbound_unexpected_and_malformed_receipts_fail_closed(key: str, replacement: object) -> None: + altered = copy.deepcopy(receipt()) + altered[key] = replacement + with pytest.raises(ObservationError): + build_summary(evidence(), altered, b"{}") + + +def test_subject_mismatch_and_unknown_decision_fail_closed() -> None: + altered = evidence() + altered["subject"]["head_sha"] = HEAD + with pytest.raises(ObservationError, match="evidence subject"): + build_summary(altered, receipt(), b"{}") + with pytest.raises(ObservationError, match="decision state"): + build_summary(evidence("authoritative_verified"), receipt(), b"{}") + + +def test_renderer_appends_without_overwriting_existing_job_summary(tmp_path: Path) -> None: + observation = tmp_path / "evidence.json" + binding = tmp_path / "binding.json" + output = tmp_path / "summary.md" + observation.write_text(json.dumps(evidence()), encoding="utf-8") + binding.write_text(json.dumps(receipt()), encoding="utf-8") + output.write_text("## Existing reviewer summary\n\n", encoding="utf-8") + + assert main([ + "--evidence", str(observation), + "--source-binding", str(binding), + "--output", str(output), + ]) == 0 + contents = output.read_text(encoding="utf-8") + assert contents.startswith("## Existing reviewer summary\n\n") + assert "advisory PR observation" in contents + assert "source revision" not in contents or MERGE in contents + assert observation.read_bytes() and binding.read_bytes() + + +def test_no_summary_written_from_unbound_input(tmp_path: Path) -> None: + observation = tmp_path / "evidence.json" + binding = tmp_path / "binding.json" + output = tmp_path / "summary.md" + observation.write_text(json.dumps(evidence("allow")), encoding="utf-8") + unbound = receipt() + unbound["input_structurally_bound_to_checkout"] = False + binding.write_text(json.dumps(unbound), encoding="utf-8") + + assert main([ + "--evidence", str(observation), + "--source-binding", str(binding), + "--output", str(output), + ]) == 2 + assert not output.exists() + + +def test_job_summary_only_runs_for_default_source_bound_advisory_pr() -> None: + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] + step = next(s for s in steps if s.get("name") == "Share scoped advisory observation in job summary") + condition = step["if"] + for guard in ( + "github.event_name == 'pull_request'", + "inputs.mode == 'advisory'", + "inputs.verification-manifest == ''", + "inputs.changed-files == ''", + "inputs.use-check == 'true'", + ): + assert guard in condition + assert 'render_source_bound_job_summary.py' in step["run"] + assert '"$GITHUB_STEP_SUMMARY"' in step["run"] + assert "GITHUB_TOKEN" not in step.get("env", {}) + assert "GITHUB_TOKEN" not in step["run"] + assert "check-runs" not in step["run"] From 0097acaecf1a0d8841748fca38ec7e3b5de7f09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:34:33 -0700 Subject: [PATCH 081/104] test(product): assert exact source and evidence disclaimers --- tests/test_source_scoped_job_summary.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_source_scoped_job_summary.py b/tests/test_source_scoped_job_summary.py index 10d9c1ca5..6b381bbd3 100644 --- a/tests/test_source_scoped_job_summary.py +++ b/tests/test_source_scoped_job_summary.py @@ -60,10 +60,10 @@ def test_shareable_summary_is_precisely_scoped_and_identifiable() -> None: for id_ in (BASE, HEAD, MERGE, DIFF): assert id_ in markdown assert hashlib.sha256(raw).hexdigest() in markdown - assert "not independently authenticated" not in markdown or "independently authenticated" in markdown + assert "neither the event nor the evidence is independently authenticated" in markdown assert "An `allow` is not permission to merge." in markdown assert "not signed into the evidence envelope" in markdown - assert "source-to-model" not in markdown or "source" in markdown + assert "supported-source completeness" in markdown @pytest.mark.parametrize( @@ -115,7 +115,7 @@ def test_renderer_appends_without_overwriting_existing_job_summary(tmp_path: Pat contents = output.read_text(encoding="utf-8") assert contents.startswith("## Existing reviewer summary\n\n") assert "advisory PR observation" in contents - assert "source revision" not in contents or MERGE in contents + assert MERGE in contents assert observation.read_bytes() and binding.read_bytes() From 127ae8f42168bc5c74bfd53edc60c0b3bba9a1b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:35:04 -0700 Subject: [PATCH 082/104] docs(product): portable no-write summary and observed adoption loop --- docs/SOURCE_SCOPED_JOB_SUMMARY.md | 85 +++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 docs/SOURCE_SCOPED_JOB_SUMMARY.md diff --git a/docs/SOURCE_SCOPED_JOB_SUMMARY.md b/docs/SOURCE_SCOPED_JOB_SUMMARY.md new file mode 100644 index 000000000..e00f6251a --- /dev/null +++ b/docs/SOURCE_SCOPED_JOB_SUMMARY.md @@ -0,0 +1,85 @@ +# Read-only source-scoped PR summary — developer preview + +**Status: unmerged development feature, not part of published `v1.2.1`.** +The historical Action does not contain this feature. This document records a +candidate [read-only summary workflow](https://github.com/fraware/open-verification-kernel/pull/227), +pending its exact-commit CI, source-path review, release governance and a +consenting independent adoption experiment. No production relying authority +or externally calibrated verification claim is established. + +## Reviewer experience + +A consenting repository enables the OVK composite Action on an ordinary +`pull_request` workflow with: + +- a full-SHA-reviewed Action revision and `actions/checkout` at the normal + GitHub merge-preview event SHA, with a sufficiently deep checkout; +- `mode: advisory`, `use-check: "true"`, **no** explicit `changed-files` + or `verification-manifest` input, `emit-check: "false"`, + `post-comment: "false"`; +- job `permissions: {contents: read}` and no untrusted PR secrets. + +Only in that bounded default path, the Action reads the local +`ovk-evidence.json` and `ovk-source-binding.json`. It rejects absent, +unbound, structurally inconsistent or mismatched source identifiers before +appending a concise summary to GitHub's native `GITHUB_STEP_SUMMARY` file. +This uses the existing workflow's job-summary presentation; no PR comment, +GitHub Check-write permission, third-party service, or hosted account is +needed. + +The summary includes the *recorded scoped* checker decision, the observed +merge-preview, original PR-head and base SHAs, SHA-256 of the observed diff +and of the exact local evidence bytes. It states that source/event +authenticity, complete source-to-model correspondence, signed envelope +binding, production reliance, and whole-PR security **are not established**. +A recorded `allow` never authorizes merging. No user-controlled source +snippet or raw code is copied into the summary. + +The job summary is a *review-navigation and evidence-identification artifact*, +not a signed certificate. Its identifiers are useful for comparing local +reports and locating the underlying artifacts. A SHA-256 digest is tamper +evidence only relative to a trusted expected digest; it does not establish +the authenticity of the evidence or the GitHub runner. + +## Reuse across repositories and CI systems + +The renderer itself uses only Python's standard library: + +```bash +python scripts/render_source_bound_job_summary.py \ + --evidence ovk-evidence.json \ + --source-binding ovk-source-binding.json \ + --output advisory-summary.md +``` + +The input files must come from the same ordinary, internally prepared +source-bound OVK PR run. Copying fixtures, passing caller-selected diffs, +relabeling PR heads, or using a multi-lane manifest is deliberately refused. +The renderer appends instead of overwriting an existing summary, so +multiple independent tools in the same CI job retain their own sections. +The file can be used in a different CI provider only if the exact local +observation contract is reproduced and checked; the script itself does not +authenticate third-party events. + +Use this first as a **read-only advisory preview**, strictly with maintainer +consent and human review. Existing public release controls in issue #25 and +source-model/refinement qualification in issue #195 remain separate. + +## Distribution and utility measurement + +Track the adoption chain prospectively, with source evidence and no inflated +marketing counts: + +1. consented outside-repository installation and setup time; +2. first natural PR with a complete, intelligible, source-identified + observation and explicit unsupported/unknown cases; +3. an independent reviewer reading the summary and the underlying evidence, + including reviewer minutes, useful findings and erroneous alerts; +4. a second natural PR and independent teammate adoption; +5. legitimate cross-repository reuse of an explicitly specified claim, with + applicability conditions and independent review, not a copy-pasted PASS. + +Treat the job summary's page views and GitHub stars as distribution indicators, +not as formal assurance, effective vulnerability detection, or economic value. +A pilot with no qualifying claims is a valid null result; report coverage and +attrition rather than claiming a perfect false-positive or false-negative rate. From db3e2b7279d1c4acb9ddea70f4ddb68cb86d61e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:36:36 -0700 Subject: [PATCH 083/104] security(summary): bind observed diff bytes and checkout tree to reader-facing summary --- scripts/render_source_bound_job_summary.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/render_source_bound_job_summary.py b/scripts/render_source_bound_job_summary.py index 56259534b..b20ddaf27 100644 --- a/scripts/render_source_bound_job_summary.py +++ b/scripts/render_source_bound_job_summary.py @@ -37,7 +37,12 @@ def _json_object(path: Path) -> tuple[dict[str, Any], bytes]: return payload, raw -def build_summary(evidence: dict[str, Any], receipt: dict[str, Any], evidence_bytes: bytes) -> str: +def build_summary( + evidence: dict[str, Any], + receipt: dict[str, Any], + evidence_bytes: bytes, + diff_bytes: bytes, +) -> str: """Build a non-authoritative advisory report, rejecting unbound or stale input.""" if receipt.get("schema_version") != SCHEMA: raise ObservationError("unknown source observation contract") @@ -55,15 +60,18 @@ def build_summary(evidence: dict[str, Any], receipt: dict[str, Any], evidence_by raise ObservationError("unexpected reliance status") checkout = receipt.get("evaluated_checkout_commit_sha") + tree = receipt.get("evaluated_checkout_tree_sha") subject = receipt.get("evidence_subject_sha") head = receipt.get("event_pr_head_sha") base = receipt.get("event_base_sha") diff_hash = receipt.get("input_diff_sha256") - fields = {"checkout": checkout, "subject": subject, "head": head, "base": base} + fields = {"checkout": checkout, "tree": tree, "subject": subject, "head": head, "base": base} if any(not isinstance(v, str) or SHA40.fullmatch(v) is None for v in fields.values()): raise ObservationError("missing or malformed source revision") if not isinstance(diff_hash, str) or DIGEST64.fullmatch(diff_hash) is None: raise ObservationError("missing or malformed diff digest") + if hashlib.sha256(diff_bytes).hexdigest() != diff_hash: + raise ObservationError("observed diff bytes disagree with receipt digest") if checkout != subject: raise ObservationError("receipt subject does not match evaluated checkout") evidence_subject = evidence.get("subject") @@ -115,6 +123,7 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--evidence", type=Path, default=Path("ovk-evidence.json")) parser.add_argument("--source-binding", type=Path, default=Path("ovk-source-binding.json")) + parser.add_argument("--diff", type=Path, default=Path(".ovk-pr.diff")) parser.add_argument( "--output", type=Path, default=Path(os.environ["GITHUB_STEP_SUMMARY"]) if os.environ.get("GITHUB_STEP_SUMMARY") else None, @@ -127,7 +136,8 @@ def main(argv: list[str] | None = None) -> int: try: evidence, evidence_bytes = _json_object(args.evidence) receipt, _ = _json_object(args.source_binding) - summary = build_summary(evidence, receipt, evidence_bytes) + diff_bytes = args.diff.read_bytes() + summary = build_summary(evidence, receipt, evidence_bytes, diff_bytes) # The host owns GITHUB_STEP_SUMMARY; append so this Action composes # with earlier steps and multiple invocations without clobbering them. with args.output.open("a", encoding="utf-8", newline="\n") as output: From 2b55d4e4dcffba3c267559559b8e034627fb4317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:36:54 -0700 Subject: [PATCH 084/104] test(summary): reject tampered or unavailable PR diff bytes --- tests/test_source_scoped_job_summary.py | 42 +++++++++++++++++++++---- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/tests/test_source_scoped_job_summary.py b/tests/test_source_scoped_job_summary.py index 6b381bbd3..c4880d2bf 100644 --- a/tests/test_source_scoped_job_summary.py +++ b/tests/test_source_scoped_job_summary.py @@ -21,7 +21,9 @@ BASE = "a" * 40 HEAD = "b" * 40 MERGE = "c" * 40 -DIFF = "d" * 64 +TREE = "f" * 40 +DIFF_BYTES = b"diff --git a/sample.py b/sample.py\\n+authorized = True\\n" +DIFF = hashlib.sha256(DIFF_BYTES).hexdigest() def receipt() -> dict: @@ -31,6 +33,7 @@ def receipt() -> dict: "event_base_sha": BASE, "event_pr_head_sha": HEAD, "evaluated_checkout_commit_sha": MERGE, + "evaluated_checkout_tree_sha": TREE, "evidence_subject_sha": MERGE, "input_comparison": "event_base_to_verified_merge_preview", "input_diff_sha256": DIFF, @@ -54,10 +57,10 @@ def evidence(state: str = "allow") -> dict: def test_shareable_summary_is_precisely_scoped_and_identifiable() -> None: data = evidence("block") raw = json.dumps(data, sort_keys=True).encode() - markdown = build_summary(data, receipt(), raw) + markdown = build_summary(data, receipt(), raw, DIFF_BYTES) assert "advisory PR observation" in markdown assert "**Recorded scoped decision:** `block`" in markdown - for id_ in (BASE, HEAD, MERGE, DIFF): + for id_ in (BASE, HEAD, MERGE, TREE, DIFF): assert id_ in markdown assert hashlib.sha256(raw).hexdigest() in markdown assert "neither the event nor the evidence is independently authenticated" in markdown @@ -79,6 +82,7 @@ def test_shareable_summary_is_precisely_scoped_and_identifiable() -> None: ("evaluated_checkout_commit_sha", "wrong"), ("event_pr_head_sha", "missing"), ("event_base_sha", "BAD"), + ("evaluated_checkout_tree_sha", "missing"), ("input_diff_sha256", "not-a-digest"), ("evidence_subject_sha", BASE), ], @@ -87,21 +91,23 @@ def test_unbound_unexpected_and_malformed_receipts_fail_closed(key: str, replace altered = copy.deepcopy(receipt()) altered[key] = replacement with pytest.raises(ObservationError): - build_summary(evidence(), altered, b"{}") + build_summary(evidence(), altered, b"{}", DIFF_BYTES) def test_subject_mismatch_and_unknown_decision_fail_closed() -> None: altered = evidence() altered["subject"]["head_sha"] = HEAD with pytest.raises(ObservationError, match="evidence subject"): - build_summary(altered, receipt(), b"{}") + build_summary(altered, receipt(), b"{}", DIFF_BYTES) with pytest.raises(ObservationError, match="decision state"): - build_summary(evidence("authoritative_verified"), receipt(), b"{}") + build_summary(evidence("authoritative_verified"), receipt(), b"{}", DIFF_BYTES) def test_renderer_appends_without_overwriting_existing_job_summary(tmp_path: Path) -> None: observation = tmp_path / "evidence.json" binding = tmp_path / "binding.json" + diff = tmp_path / "actual-pr.diff" + diff.write_bytes(DIFF_BYTES) output = tmp_path / "summary.md" observation.write_text(json.dumps(evidence()), encoding="utf-8") binding.write_text(json.dumps(receipt()), encoding="utf-8") @@ -110,6 +116,7 @@ def test_renderer_appends_without_overwriting_existing_job_summary(tmp_path: Pat assert main([ "--evidence", str(observation), "--source-binding", str(binding), + "--diff", str(diff), "--output", str(output), ]) == 0 contents = output.read_text(encoding="utf-8") @@ -122,6 +129,8 @@ def test_renderer_appends_without_overwriting_existing_job_summary(tmp_path: Pat def test_no_summary_written_from_unbound_input(tmp_path: Path) -> None: observation = tmp_path / "evidence.json" binding = tmp_path / "binding.json" + diff = tmp_path / "actual-pr.diff" + diff.write_bytes(DIFF_BYTES) output = tmp_path / "summary.md" observation.write_text(json.dumps(evidence("allow")), encoding="utf-8") unbound = receipt() @@ -131,6 +140,7 @@ def test_no_summary_written_from_unbound_input(tmp_path: Path) -> None: assert main([ "--evidence", str(observation), "--source-binding", str(binding), + "--diff", str(diff), "--output", str(output), ]) == 2 assert not output.exists() @@ -154,3 +164,23 @@ def test_job_summary_only_runs_for_default_source_bound_advisory_pr() -> None: assert "GITHUB_TOKEN" not in step.get("env", {}) assert "GITHUB_TOKEN" not in step["run"] assert "check-runs" not in step["run"] + + +def test_modified_diff_bytes_cannot_be_described_by_old_receipt() -> None: + with pytest.raises(ObservationError, match="diff bytes"): + build_summary(evidence(), receipt(), b"{}", DIFF_BYTES + b"unexpected") + + +def test_missing_source_diff_never_creates_report(tmp_path: Path) -> None: + observation = tmp_path / "evidence.json" + binding = tmp_path / "binding.json" + destination = tmp_path / "summary.md" + observation.write_text(json.dumps(evidence()), encoding="utf-8") + binding.write_text(json.dumps(receipt()), encoding="utf-8") + assert main([ + "--evidence", str(observation), + "--source-binding", str(binding), + "--diff", str(tmp_path / "absent.diff"), + "--output", str(destination), + ]) == 2 + assert not destination.exists() From 923ee9692277aa81c0248629d02faea79938d9b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:37:12 -0700 Subject: [PATCH 085/104] feat(action): verify actual binary input bytes before rendering summary --- action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/action.yml b/action.yml index 26d811cbb..64257f25f 100644 --- a/action.yml +++ b/action.yml @@ -270,6 +270,7 @@ runs: python "${{ github.action_path }}/scripts/render_source_bound_job_summary.py" \ --evidence ovk-evidence.json \ --source-binding ovk-source-binding.json \ + --diff .ovk-pr.diff \ --output "$GITHUB_STEP_SUMMARY" - name: Emit OVK GitHub check (check path) id: emit-check-run From 3ecf77b558f30cf1a112dab91dc9818488cf4e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:37:21 -0700 Subject: [PATCH 086/104] docs(product): make exact binary diff verification explicit --- docs/SOURCE_SCOPED_JOB_SUMMARY.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/SOURCE_SCOPED_JOB_SUMMARY.md b/docs/SOURCE_SCOPED_JOB_SUMMARY.md index e00f6251a..353bafd3b 100644 --- a/docs/SOURCE_SCOPED_JOB_SUMMARY.md +++ b/docs/SOURCE_SCOPED_JOB_SUMMARY.md @@ -49,11 +49,15 @@ The renderer itself uses only Python's standard library: python scripts/render_source_bound_job_summary.py \ --evidence ovk-evidence.json \ --source-binding ovk-source-binding.json \ + --diff .ovk-pr.diff \ --output advisory-summary.md ``` -The input files must come from the same ordinary, internally prepared -source-bound OVK PR run. Copying fixtures, passing caller-selected diffs, +The evidence, source receipt, and **exact binary diff bytes** must come +from the same ordinary, internally prepared source-bound OVK PR run. The +renderer recomputes the input diff digest and refuses a changed or missing +diff; it also checks the complete source tree SHA field and agreement of +the evidence subject with the observed merge preview. Copying fixtures, passing caller-selected diffs, relabeling PR heads, or using a multi-lane manifest is deliberately refused. The renderer appends instead of overwriting an existing summary, so multiple independent tools in the same CI job retain their own sections. From e582a113cc4b4c272856d9fb3fd6e4862a5716b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:37:43 -0700 Subject: [PATCH 087/104] test(summary): use real newline-bearing diffs and assert byte-source handoff --- tests/test_source_scoped_job_summary.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_source_scoped_job_summary.py b/tests/test_source_scoped_job_summary.py index c4880d2bf..be6b58468 100644 --- a/tests/test_source_scoped_job_summary.py +++ b/tests/test_source_scoped_job_summary.py @@ -22,7 +22,7 @@ HEAD = "b" * 40 MERGE = "c" * 40 TREE = "f" * 40 -DIFF_BYTES = b"diff --git a/sample.py b/sample.py\\n+authorized = True\\n" +DIFF_BYTES = b"diff --git a/sample.py b/sample.py\n+authorized = True\n" DIFF = hashlib.sha256(DIFF_BYTES).hexdigest() @@ -160,6 +160,7 @@ def test_job_summary_only_runs_for_default_source_bound_advisory_pr() -> None: ): assert guard in condition assert 'render_source_bound_job_summary.py' in step["run"] + assert "--diff .ovk-pr.diff" in step["run"] assert '"$GITHUB_STEP_SUMMARY"' in step["run"] assert "GITHUB_TOKEN" not in step.get("env", {}) assert "GITHUB_TOKEN" not in step["run"] From 5b6423acf95c1dc1ed87010b4f8eb530ecd11dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:38:26 -0700 Subject: [PATCH 088/104] test(action): assert no PR comments from source-unbound manifests --- tests/test_action_combined_identity_contract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_action_combined_identity_contract.py b/tests/test_action_combined_identity_contract.py index 07a0b198c..e49a592db 100644 --- a/tests/test_action_combined_identity_contract.py +++ b/tests/test_action_combined_identity_contract.py @@ -61,7 +61,8 @@ def test_unbound_manual_diff_cannot_be_published_by_default_check_path() -> None comment = by_name["Post OVK pull-request comment"] assert "inputs.changed-files == ''" in check["if"] assert "inputs.changed-files == ''" in comment["if"] - assert "inputs.verification-manifest != ''" in comment["if"] + # Explicit manifests bypass source binding; public comments are forbidden. + assert "inputs.verification-manifest == ''" in comment["if"] upload = by_name["Upload OVK artifacts"] assert "ovk-source-binding.json" in upload["with"]["path"] From 90f0a4492056149fd57c84502d2ebea80d2d8231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:39:15 -0700 Subject: [PATCH 089/104] fix(summary): expose validated Git tree identity for cross-revision comparison --- scripts/render_source_bound_job_summary.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/render_source_bound_job_summary.py b/scripts/render_source_bound_job_summary.py index b20ddaf27..54b846fdd 100644 --- a/scripts/render_source_bound_job_summary.py +++ b/scripts/render_source_bound_job_summary.py @@ -103,6 +103,7 @@ def build_summary( "| Observation | Exact value |\n" "| --- | --- |\n" f"| Evaluated GitHub merge-preview commit | `{checkout}` |\n" + f"| Observed Git tree identity | `{tree}` |\n" f"| Original contributor PR head | `{head}` |\n" f"| Event base revision | `{base}` |\n" f"| Observed binary diff SHA-256 | `{diff_hash}` |\n" From ae1831f4e1a04dda7bd9e725daed58f1b7760b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:18:52 -0700 Subject: [PATCH 090/104] fix(source): validate actual PR merge subject before evidence emission --- ovk/core/source_binding_identity.py | 108 ++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 ovk/core/source_binding_identity.py diff --git a/ovk/core/source_binding_identity.py b/ovk/core/source_binding_identity.py new file mode 100644 index 000000000..c7778bd1f --- /dev/null +++ b/ovk/core/source_binding_identity.py @@ -0,0 +1,108 @@ +"""Admit a *local observed* merge-preview subject for the OVK check CLI. + +The Action's source-input preparer has already checked the current Git checkout +and generated a receipt. This module independently compares that receipt to +the current checkout, the current PR event, and exact diff bytes before the +event-derived PR-head value may be replaced as the evidence subject. + +These are local structural checks, not authenticated GitHub provenance or +source-to-IR correctness. Never treat their success as a signed certificate. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from pathlib import Path +from typing import Any + + +SHA = re.compile(r"^[0-9a-f]{40}$") +DIGEST = re.compile(r"^[0-9a-f]{64}$") +SCHEMA = "ovk.pr_source_input_binding.v1-development" +COMPARISON = "event_base_to_verified_merge_preview" + + +class SourceSubjectMismatch(ValueError): + pass + + +def _read_object(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise SourceSubjectMismatch(f"expected object in {path.name}") + return data + + +def _sha(value: Any, name: str) -> str: + if not isinstance(value, str) or SHA.fullmatch(value) is None: + raise SourceSubjectMismatch(f"missing or malformed {name}") + return value + + +def _git_object(revspec: str) -> str: + try: + result = subprocess.run( + ["git", "rev-parse", "--verify", revspec], + check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, timeout=10, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise SourceSubjectMismatch("unable to inspect checked-out Git source") from exc + if result.returncode != 0: + raise SourceSubjectMismatch("unable to inspect checked-out Git source") + return _sha(result.stdout.strip(), "checked-out Git object") + + +def require_bound_merge_preview_subject( + *, receipt_path: Path, diff_path: Path, event_path: Path, + repo: str, head_sha: str, base_sha: str | None, +) -> None: + """Fail closed unless all locally observed source identities agree.""" + receipt = _read_object(receipt_path) + event = _read_object(event_path) + if receipt.get("schema_version") != SCHEMA or receipt.get("event_name") != "pull_request": + raise SourceSubjectMismatch("unsupported PR source observation contract") + if receipt.get("input_comparison") != COMPARISON: + raise SourceSubjectMismatch("input comparison is not bound to PR merge preview") + if receipt.get("input_structurally_bound_to_checkout") is not True: + raise SourceSubjectMismatch("change input was not structurally bound to checkout") + for key in ("independent_event_authentication", "bound_into_signed_evidence_envelope", + "production_reliance_authorized"): + if receipt.get(key) is not False: + raise SourceSubjectMismatch(f"unexpected reliance field: {key}") + + checkout = _sha(receipt.get("evaluated_checkout_commit_sha"), "checkout commit") + tree = _sha(receipt.get("evaluated_checkout_tree_sha"), "checkout tree") + subject = _sha(receipt.get("evidence_subject_sha"), "receipt evidence subject") + event_base = _sha(receipt.get("event_base_sha"), "event base") + event_head = _sha(receipt.get("event_pr_head_sha"), "event PR head") + requested = _sha(head_sha, "explicit requested evidence subject") + if subject != checkout or requested != checkout: + raise SourceSubjectMismatch("explicit evidence subject is not checked-out merge preview") + if _git_object("HEAD") != checkout or _git_object("HEAD^{tree}") != tree: + raise SourceSubjectMismatch("checked-out Git object differs from source receipt") + + pr = event.get("pull_request") + ev_repo = event.get("repository") + if not isinstance(pr, dict) or not isinstance(ev_repo, dict): + raise SourceSubjectMismatch("GitHub pull_request event identity is missing") + ev_base, ev_head = pr.get("base"), pr.get("head") + if not isinstance(ev_base, dict) or not isinstance(ev_head, dict): + raise SourceSubjectMismatch("GitHub event PR parent objects are missing") + if _sha(ev_base.get("sha"), "GitHub event base") != event_base: + raise SourceSubjectMismatch("GitHub event base differs from source receipt") + if _sha(ev_head.get("sha"), "GitHub event head") != event_head: + raise SourceSubjectMismatch("GitHub event PR head differs from source receipt") + if ev_repo.get("full_name") != repo: + raise SourceSubjectMismatch("GitHub event repository differs from requested repository") + if base_sha is not None and base_sha != event_base: + raise SourceSubjectMismatch("caller base differs from checked PR base") + + digest = receipt.get("input_diff_sha256") + if not isinstance(digest, str) or DIGEST.fullmatch(digest) is None: + raise SourceSubjectMismatch("missing or malformed exact diff digest") + if hashlib.sha256(diff_path.read_bytes()).hexdigest() != digest: + raise SourceSubjectMismatch("actual input diff bytes differ from source receipt") From d6e3009b49b2100449d8a44dd0e0f14bb00589e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:19:09 -0700 Subject: [PATCH 091/104] fix(context): retain explicitly validated merge-preview subject without losing PR event head --- ovk/core/context.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ovk/core/context.py b/ovk/core/context.py index c9cfb222e..87069f9e1 100644 --- a/ovk/core/context.py +++ b/ovk/core/context.py @@ -204,6 +204,7 @@ def build_repository_context( repo: str = "unknown/repo", head_sha: str = "unknown", base_sha: str | None = None, + preserve_explicit_subject: bool = False, ) -> RepositoryContext: """Build repository context from PR metadata and trusted policy materials.""" files = changed_files or [] @@ -214,7 +215,16 @@ def build_repository_context( defaults = metadata_to_self_protection_defaults(github) actor_type = str(defaults.get("actor_type", "unknown")) repo = github.repository or repo - head_sha = github.head_sha or head_sha + # A separately checked source-binding receipt permits an observed + # merge-preview to be the subject of a default PR diff. Ordinary + # event-based callers retain the historical PR-head convention. + if preserve_explicit_subject: + if not head_sha or head_sha == "unknown": + raise ValueError("bound source subject requires explicit checkout SHA") + branch_metadata["event_pr_head_sha"] = github.head_sha + branch_metadata["source_subject_origin"] = "locally_bound_merge_preview" + else: + head_sha = github.head_sha or head_sha base_sha = github.base_sha or base_sha branch_metadata["github"] = defaults if check_metadata_path is not None and check_metadata_path.exists(): From b7b94f086017a17153bbbeb46723954e80f36a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:19:23 -0700 Subject: [PATCH 092/104] fix(source): propagate structurally checked merge subject through kernel --- ovk/core/kernel.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ovk/core/kernel.py b/ovk/core/kernel.py index 85965b572..a43bdde89 100644 --- a/ovk/core/kernel.py +++ b/ovk/core/kernel.py @@ -90,6 +90,7 @@ def execute_kernel( repo: str = "unknown/repo", head_sha: str = "unknown", base_sha: str | None = None, + preserve_explicit_subject: bool = False, budget: VerificationBudget | None = None, cache_dir: Path | None = DEFAULT_CACHE_DIR, use_cache: bool = True, @@ -113,6 +114,7 @@ def execute_kernel( repo=repo, head_sha=head_sha, base_sha=base_sha, + preserve_explicit_subject=preserve_explicit_subject, ) compiler = ObligationCompilerRegistry.default() From 7c85575f4928bc8c321d2b4bf65f3d471dea54ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:19:26 -0700 Subject: [PATCH 093/104] fix(source): propagate structurally checked merge subject through kernel --- ovk/core/check.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ovk/core/check.py b/ovk/core/check.py index 0d7c5b952..5e66a5732 100644 --- a/ovk/core/check.py +++ b/ovk/core/check.py @@ -51,6 +51,7 @@ def run_check( repo: str = "unknown/repo", head_sha: str = "unknown", base_sha: str | None = None, + preserve_explicit_subject: bool = False, cache_dir: Path | None = None, use_cache: bool = True, parallel: bool = True, @@ -67,6 +68,7 @@ def run_check( repo=repo, head_sha=head_sha, base_sha=base_sha, + preserve_explicit_subject=preserve_explicit_subject, cache_dir=cache_dir if cache_dir is not None else DEFAULT_CACHE_DIR, use_cache=use_cache, parallel=parallel, From 27d5c8c75c8c6378ded07d7ef8cc2c62008b2d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:19:46 -0700 Subject: [PATCH 094/104] fix(cli): select merge-preview evidence subject only after local receipt/event/diff validation --- ovk/cli.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ovk/cli.py b/ovk/cli.py index bcf3970f4..2aad5a5a4 100644 --- a/ovk/cli.py +++ b/ovk/cli.py @@ -726,11 +726,34 @@ def check( strict: bool = typer.Option(False, help="Enforce non-zero exit codes (overrides advisory)."), no_cache: bool = typer.Option(False, help="Disable lane result cache."), format: str = typer.Option("md", help="Output format: md, json, or github."), + source_binding: Optional[Path] = typer.Option( + None, "--source-binding", + help="Local PR merge-preview source receipt; checked against Git/event/diff before choosing evidence subject.", + ), ) -> None: """Infer, compile, and verify affected lanes for a PR diff.""" enforce_exit = strict or not advisory input_path = diff or changed_files files, diff_text = load_diff_or_changed_files(input_path) + if source_binding is not None: + if input_path is None or diff_text is None or github_event is None: + raise typer.BadParameter("source binding requires an actual unified diff and GitHub PR event") + from ovk.core.source_binding_identity import ( + SourceSubjectMismatch, + require_bound_merge_preview_subject, + ) + + try: + require_bound_merge_preview_subject( + receipt_path=source_binding, + diff_path=input_path, + event_path=github_event, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + except (SourceSubjectMismatch, OSError, ValueError) as exc: + raise typer.BadParameter(f"refusing unbound evidence subject: {exc}") from exc meta = read_json_file(metadata) if metadata else None result = run_check( changed_files=files, @@ -741,6 +764,7 @@ def check( repo=repo, head_sha=head_sha, base_sha=base_sha, + preserve_explicit_subject=source_binding is not None, use_cache=not no_cache, ) output_dir.mkdir(parents=True, exist_ok=True) From b7020afa50db8d6b25d52d7f1c697d2424f01cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:20:06 -0700 Subject: [PATCH 095/104] fix(action): pin check evidence subject to validated PR merge preview --- action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/action.yml b/action.yml index 64257f25f..ca4dcfcc1 100644 --- a/action.yml +++ b/action.yml @@ -230,6 +230,8 @@ runs: OVK_MODE: ${{ inputs.mode }} OVK_CHECK_METADATA: ${{ inputs.check-metadata }} OVK_GITHUB_EVENT: ${{ github.event_path }} + OVK_EVENT_NAME: ${{ github.event_name }} + OVK_CHANGED_FILES: ${{ inputs.changed-files }} OVK_REPO: ${{ github.repository }} OVK_HEAD_SHA: ${{ github.sha }} run: | @@ -241,6 +243,12 @@ runs: --head-sha "$OVK_HEAD_SHA" --output-dir . ) + if [ "$OVK_EVENT_NAME" = "pull_request" ] && [ -z "$OVK_CHANGED_FILES" ]; then + # The locally generated source receipt was checked against the + # checkout and event. Bind the evidence subject to that exact merge, + # not to the contributor PR head re-read from the event payload. + args+=(--source-binding ovk-source-binding.json) + fi if [ -n "$OVK_CHECK_METADATA" ]; then args+=(--check-metadata "$OVK_CHECK_METADATA") elif [ -f ovk-branch-metadata.json ]; then From 0fbf565af79e384df597af15fc131031ce20d0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:20:54 -0700 Subject: [PATCH 096/104] test(source): real Git PR merge must survive event metadata and reject confused subjects --- tests/test_source_bound_merge_subject.py | 168 +++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 tests/test_source_bound_merge_subject.py diff --git a/tests/test_source_bound_merge_subject.py b/tests/test_source_bound_merge_subject.py new file mode 100644 index 000000000..f1a22d758 --- /dev/null +++ b/tests/test_source_bound_merge_subject.py @@ -0,0 +1,168 @@ +"""A real merge-preview must remain the evidence subject after PR event ingestion. + +Local source checks are structural, not signature verification or generalized +source-to-model fidelity. This test creates real Git commits and a merge. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from ovk.cli import app +from ovk.core.context import build_repository_context +from ovk.core.source_binding_identity import ( + SourceSubjectMismatch, require_bound_merge_preview_subject, +) +from scripts.prepare_verified_change_input import prepare + + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="Git required") + + +def git(*args: str) -> str: + result = subprocess.run( + ["git", *args], capture_output=True, text=True, check=True, + ) + return result.stdout.strip() + + +@pytest.fixture +def merge_pr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict: + monkeypatch.chdir(tmp_path) + git("init", "-b", "main") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "Source Identity Test") + git("config", "core.autocrlf", "false") + (tmp_path / "original.txt").write_text("original\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "base") + base = git("rev-parse", "HEAD") + git("checkout", "-b", "contribution") + (tmp_path / "example.py").write_text("sink()\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "head") + pr_head = git("rev-parse", "HEAD") + git("checkout", "main") + git("merge", "--no-ff", "contribution", "-m", "merge preview") + merged = git("rev-parse", "HEAD") + assert merged != pr_head + event_path = tmp_path / "pr-event.json" + event_path.write_text(json.dumps({ + "repository": {"full_name": "pilot/repo"}, + "pull_request": { + "number": 17, + "head": {"sha": pr_head}, + "base": {"sha": base}, + }, + "sender": {"login": "contributor", "type": "User"}, + }), encoding="utf-8") + path, diff, receipt = prepare( + event_name="pull_request", github_sha=merged, base_sha=base, pr_head_sha=pr_head, + ) + assert path == ".ovk-pr.diff" + diff_path = tmp_path / path + diff_path.write_bytes(diff) + receipt_path = tmp_path / "ovk-source-binding.json" + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + return { + "base": base, "head": pr_head, "merge": merged, + "event": event_path, "diff": diff_path, + "receipt": receipt_path, + } + + +def validate(case: dict) -> None: + require_bound_merge_preview_subject( + receipt_path=case["receipt"], + diff_path=case["diff"], + event_path=case["event"], + repo="pilot/repo", + head_sha=case["merge"], + base_sha=case["base"], + ) + + +def test_real_pr_subject_receipt_and_context_agree(merge_pr: dict) -> None: + validate(merge_pr) + ordinary = build_repository_context( + changed_files=["example.py"], github_event_path=merge_pr["event"], + repo="pilot/repo", head_sha=merge_pr["merge"], + ) + assert ordinary.head_sha == merge_pr["head"] + bound = build_repository_context( + changed_files=["example.py"], github_event_path=merge_pr["event"], + repo="pilot/repo", head_sha=merge_pr["merge"], + preserve_explicit_subject=True, + ) + assert bound.head_sha == merge_pr["merge"] + assert bound.base_sha == merge_pr["base"] + assert bound.branch_metadata["event_pr_head_sha"] == merge_pr["head"] + assert bound.branch_metadata["source_subject_origin"] == "locally_bound_merge_preview" + + +@pytest.mark.parametrize("mutation", ["receipt_head", "diff", "pr_event", "checkout", "requested_head", "base"]) +def test_mismatched_source_materials_fail_closed(merge_pr: dict, mutation: str) -> None: + case = dict(merge_pr) + if mutation == "receipt_head": + receipt = json.loads(case["receipt"].read_text()) + receipt["evidence_subject_sha"] = case["head"] + case["receipt"].write_text(json.dumps(receipt)) + elif mutation == "diff": + case["diff"].write_bytes(case["diff"].read_bytes() + b"tamper") + elif mutation == "pr_event": + event = json.loads(case["event"].read_text()) + event["pull_request"]["head"]["sha"] = case["base"] + case["event"].write_text(json.dumps(event)) + elif mutation == "checkout": + git("checkout", "contribution") + elif mutation == "requested_head": + case["merge"] = case["head"] + elif mutation == "base": + case["base"] = case["head"] + with pytest.raises(SourceSubjectMismatch): + validate(case) + + +def test_cli_emits_observed_merge_subject_and_keeps_event_pr_head(merge_pr: dict, tmp_path: Path) -> None: + output = tmp_path / "outputs" + outcome = CliRunner().invoke( + app, [ + "check", "--diff", str(merge_pr["diff"]), + "--github-event", str(merge_pr["event"]), + "--source-binding", str(merge_pr["receipt"]), + "--repo", "pilot/repo", + "--head-sha", merge_pr["merge"], + "--base-sha", merge_pr["base"], + "--output-dir", str(output), + "--advisory", "--no-cache", + ], + ) + assert outcome.exit_code == 0, outcome.output + bundle = json.loads((output / "ovk-evidence.json").read_text()) + assert bundle["subject"]["head_sha"] == merge_pr["merge"] + assert bundle["subject"]["base_sha"] == merge_pr["base"] + assert bundle["subject"]["repo"] == "pilot/repo" + + +def test_cli_rejects_fake_subject_before_publishing_evidence(merge_pr: dict, tmp_path: Path) -> None: + output = tmp_path / "out-rejected" + outcome = CliRunner().invoke( + app, [ + "check", "--diff", str(merge_pr["diff"]), + "--github-event", str(merge_pr["event"]), + "--source-binding", str(merge_pr["receipt"]), + "--repo", "pilot/repo", + "--head-sha", merge_pr["head"], + "--output-dir", str(output), + "--advisory", + ], + ) + assert outcome.exit_code != 0 + assert "refusing unbound evidence subject" in outcome.output + assert not (output / "ovk-evidence.json").exists() From 350202580066469862ee000a60cae091928bae53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:22:30 -0700 Subject: [PATCH 097/104] fix(cli): admit exact-bound empty PR diffs without inventing coverage --- ovk/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ovk/cli.py b/ovk/cli.py index 2aad5a5a4..398ff2a3d 100644 --- a/ovk/cli.py +++ b/ovk/cli.py @@ -736,8 +736,11 @@ def check( input_path = diff or changed_files files, diff_text = load_diff_or_changed_files(input_path) if source_binding is not None: - if input_path is None or diff_text is None or github_event is None: - raise typer.BadParameter("source binding requires an actual unified diff and GitHub PR event") + if input_path is None or github_event is None: + raise typer.BadParameter("source binding requires an input file and GitHub PR event") + # A valid Git merge preview can have an empty diff. The byte-level + # receipt check still applies; lack of changed lines must not be + # mislabeled as a validated source-to-model PASS. from ovk.core.source_binding_identity import ( SourceSubjectMismatch, require_bound_merge_preview_subject, From 5a17d08884bfd8fffd0b3f18db2b82bf64d63391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:22:54 -0700 Subject: [PATCH 098/104] docs(source): disclose measured event override defect and bounded opt-in correction --- docs/PR_SOURCE_BINDING_V1.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/PR_SOURCE_BINDING_V1.md b/docs/PR_SOURCE_BINDING_V1.md index eb4a9dc44..0fba144cc 100644 --- a/docs/PR_SOURCE_BINDING_V1.md +++ b/docs/PR_SOURCE_BINDING_V1.md @@ -14,7 +14,8 @@ For a GitHub `pull_request`, the event reports `base.sha` and `head.sha`. The Ac 2. Requires the base commit object to be available (attempting an explicit shallow fetch otherwise), with **no fallback** to `HEAD^ HEAD` or a different comparison on fetch failure. Computes `git diff --no-ext-diff --binary event-base checkout-merge` and its SHA-256. For non-PR push, compares the checked-out first parent to the checked-out commit. A root commit uses `git show`. 3. Writes a versioned `ovk-source-binding.json` sidecar recording the two event identities, evaluated commit/tree identity, exact diff SHA-256, input comparison mode, explicit limits on authentication/relying policy and whether input was structurally checked against the checkout. 4. Rejects mismatching or malformed commit IDs, incompatible PR parentage, and `pull_request_target`. Caller-selected change input remains supported **in advisory mode only**, and the sidecar marks it `caller_supplied_unbound`. A caller-selected input is refused in strict mode. To prevent an unbound advisory fixture from becoming a misleading shareable check or comment, the Action suppresses automatic GitHub Check and PR-comment emission for caller-selected input, and emits an explicit warning. The local advisory evidence can still be inspected as an experiment. -5. Makes this sidecar available among the existing Action artifacts. Existing `ovk check` evidence and check API calls still use `github.sha`, now consistently describing the checked-out **merge preview** for the ordinary default PR input. +5. Makes this sidecar available among the existing Action artifacts. The **15 October real-PR dogfood exposed a second source-subject defect:** `build_repository_context` was overriding the Action's explicit `github.sha` argument with the contributor's `pull_request.head.sha`. The sidecar correctly identified the merge preview, while the evidence bundle incorrectly named the PR head. Earlier source-binding unit and Action dogfood tests did not compare the two outputs. This mismatch was reproduced from the uploaded Action artifact for draft PR #227 and is not retroactively repaired by PR #223/#224/#225/#226. +6. The subsequent **draft #227** adds the opt-in `ovk check --source-binding` path: validate event/base/head/merge/tree and the exact binary diff against the actual Git checkout before preserving the requested merge-preview evidence subject. Ordinary callers without the explicit binding retain the existing PR-head event interpretation. A real-Git adversarial regression and the read-only job-summary renderer check this correspondence. These are local observational checks, not an authenticated or signed subject certificate. The new test creates real base/head/merge Git commits, asserts the chosen binary diff, introduces wrong base/head/checkout identifiers, tests a contributor-only checkout, opaque inputs, `pull_request_target`, push, and failure without writing a misleading receipt. From 1d6c07ab92d87bfd60f4bbde2f78e8e7f36d93e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:01:30 -0700 Subject: [PATCH 099/104] feat(report): export reusable scoped Markdown artifact without loosening provenance checks --- scripts/render_source_bound_job_summary.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/render_source_bound_job_summary.py b/scripts/render_source_bound_job_summary.py index 54b846fdd..748bd11e6 100644 --- a/scripts/render_source_bound_job_summary.py +++ b/scripts/render_source_bound_job_summary.py @@ -130,15 +130,32 @@ def main(argv: list[str] | None = None) -> int: default=Path(os.environ["GITHUB_STEP_SUMMARY"]) if os.environ.get("GITHUB_STEP_SUMMARY") else None, help="GitHub job summary path or another explicit Markdown file", ) + parser.add_argument( + "--portable-output", type=Path, + help="Optional standalone, read-only scoped Markdown report for artifact reuse", + ) args = parser.parse_args(argv) if args.output is None: print("no summary output path", file=sys.stderr) return 2 try: + if args.portable_output is not None: + # A report is an output, never an input or a replacement for the + # runner-owned job summary. Reject collisions before any deletion. + protected = (args.evidence, args.source_binding, args.diff, args.output) + if args.portable_output.resolve() in {p.resolve() for p in protected}: + raise ObservationError("portable output aliases source material or job summary") + if args.portable_output.is_symlink() or args.portable_output.is_dir(): + raise ObservationError("portable output must be an ordinary file path") + # If validation fails, a prior report must not survive to the + # always-running artifact uploader as if it belonged to this run. + args.portable_output.unlink(missing_ok=True) evidence, evidence_bytes = _json_object(args.evidence) receipt, _ = _json_object(args.source_binding) diff_bytes = args.diff.read_bytes() summary = build_summary(evidence, receipt, evidence_bytes, diff_bytes) + if args.portable_output is not None: + args.portable_output.write_text(summary + "\n", encoding="utf-8", newline="\n") # The host owns GITHUB_STEP_SUMMARY; append so this Action composes # with earlier steps and multiple invocations without clobbering them. with args.output.open("a", encoding="utf-8", newline="\n") as output: From 05d67eabd15be3b390ad820060cd5df0a75b3a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:01:40 -0700 Subject: [PATCH 100/104] feat(action): upload safe human-shareable PR report in advisory runs --- action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index ca4dcfcc1..45c36da9c 100644 --- a/action.yml +++ b/action.yml @@ -95,6 +95,7 @@ runs: rm -f -- ovk-evidence.json ovk-pr-comment.md ovk-attestation.json rm -f -- ovk-artifact-manifest.json ovk-evidence-quality.json rm -f -- ovk-provenance.json ovk-attestation-envelope.json + rm -f -- ovk-advisory-pr-summary.md - name: Cache pip packages uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: @@ -279,7 +280,8 @@ runs: --evidence ovk-evidence.json \ --source-binding ovk-source-binding.json \ --diff .ovk-pr.diff \ - --output "$GITHUB_STEP_SUMMARY" + --output "$GITHUB_STEP_SUMMARY" \ + --portable-output ovk-advisory-pr-summary.md - name: Emit OVK GitHub check (check path) id: emit-check-run if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest == '' && inputs.use-check == 'true' && inputs.changed-files == '' }} @@ -477,6 +479,7 @@ runs: path: | ovk-evidence.json ovk-pr-comment.md + ovk-advisory-pr-summary.md ovk-attestation.json ovk-artifact-manifest.json ovk-evidence-quality.json From 6cec23e1408f18914680502f0eec11346830eef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:02:02 -0700 Subject: [PATCH 101/104] test(report): portable report equivalence, stale suppression, and source alias guard --- tests/test_source_scoped_job_summary.py | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/test_source_scoped_job_summary.py b/tests/test_source_scoped_job_summary.py index be6b58468..c862e03ec 100644 --- a/tests/test_source_scoped_job_summary.py +++ b/tests/test_source_scoped_job_summary.py @@ -185,3 +185,84 @@ def test_missing_source_diff_never_creates_report(tmp_path: Path) -> None: "--output", str(destination), ]) == 2 assert not destination.exists() + + +def test_portable_report_matches_append_only_job_summary(tmp_path: Path) -> None: + evidence_path = tmp_path / "ovk-evidence.json" + receipt_path = tmp_path / "ovk-source-binding.json" + diff_path = tmp_path / ".ovk-pr.diff" + job_summary = tmp_path / "GITHUB_STEP_SUMMARY" + portable = tmp_path / "ovk-advisory-pr-summary.md" + evidence_path.write_text(json.dumps(evidence("allow")), encoding="utf-8") + receipt_path.write_text(json.dumps(receipt()), encoding="utf-8") + diff_path.write_bytes(DIFF_BYTES) + job_summary.write_text("## First tool\n\n", encoding="utf-8") + rc = main([ + "--evidence", str(evidence_path), "--source-binding", str(receipt_path), + "--diff", str(diff_path), "--output", str(job_summary), + "--portable-output", str(portable), + ]) + assert rc == 0 + expected = build_summary(evidence(), receipt(), evidence_path.read_bytes(), DIFF_BYTES) + assert portable.read_text(encoding="utf-8") == expected + "\n" + assert job_summary.read_text(encoding="utf-8") == "## First tool\n\n" + expected + "\n" + assert "An `allow` is not permission to merge" in portable.read_text(encoding="utf-8") + assert "independently authenticated" in portable.read_text(encoding="utf-8") + assert "ovk-advisory-pr-summary" not in evidence_path.read_text(encoding="utf-8") + + +def test_bad_source_does_not_retain_preexisting_shareable_report(tmp_path: Path) -> None: + evidence_path = tmp_path / "ovk-evidence.json" + receipt_path = tmp_path / "ovk-source-binding.json" + diff_path = tmp_path / ".ovk-pr.diff" + job_summary = tmp_path / "GITHUB_STEP_SUMMARY" + portable = tmp_path / "ovk-advisory-pr-summary.md" + evidence_path.write_text(json.dumps(evidence()), encoding="utf-8") + receipt_path.write_text(json.dumps(receipt()), encoding="utf-8") + diff_path.write_bytes(DIFF_BYTES + b"tampered") + portable.write_text("UNSAFE STALE ALLOW", encoding="utf-8") + assert main([ + "--evidence", str(evidence_path), "--source-binding", str(receipt_path), + "--diff", str(diff_path), "--output", str(job_summary), + "--portable-output", str(portable), + ]) == 2 + assert not portable.exists() + assert not job_summary.exists() + + +@pytest.mark.parametrize("alias", ["evidence", "receipt", "diff", "summary"]) +def test_portable_report_cannot_overwrite_verification_input( + tmp_path: Path, alias: str, +) -> None: + paths = { + "evidence": tmp_path / "evidence.json", + "receipt": tmp_path / "receipt.json", + "diff": tmp_path / "change.diff", + "summary": tmp_path / "job-summary.md", + } + paths["evidence"].write_text(json.dumps(evidence()), encoding="utf-8") + paths["receipt"].write_text(json.dumps(receipt()), encoding="utf-8") + paths["diff"].write_bytes(DIFF_BYTES) + before = paths[alias].read_bytes() if paths[alias].exists() else None + assert main([ + "--evidence", str(paths["evidence"]), + "--source-binding", str(paths["receipt"]), + "--diff", str(paths["diff"]), + "--output", str(paths["summary"]), + "--portable-output", str(paths[alias]), + ]) == 2 + after = paths[alias].read_bytes() if paths[alias].exists() else None + assert before == after + + +def test_action_uploads_scoped_report_without_uploading_raw_diff(tmp_path: Path) -> None: + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] + summary = next(s for s in steps if s.get("name") == "Share scoped advisory observation in job summary") + assert "--portable-output ovk-advisory-pr-summary.md" in summary["run"] + assert "inputs.changed-files == ''" in summary["if"] + assert "inputs.verification-manifest == ''" in summary["if"] + upload = next(s for s in steps if s.get("name") == "Upload OVK artifacts") + assert "ovk-advisory-pr-summary.md" in upload["with"]["path"] + cleanup = next(s for s in steps if s.get("name") == "Invalidate stale generated Action outputs") + assert "ovk-advisory-pr-summary.md" in cleanup["run"] From fe49235b669e7c18b8fa79268e2c9a1d3dc3962f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:02:22 -0700 Subject: [PATCH 102/104] docs(report): portable shareable summary with explicit unsigned limitations --- docs/SOURCE_SCOPED_JOB_SUMMARY.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/SOURCE_SCOPED_JOB_SUMMARY.md b/docs/SOURCE_SCOPED_JOB_SUMMARY.md index 353bafd3b..96c0954ff 100644 --- a/docs/SOURCE_SCOPED_JOB_SUMMARY.md +++ b/docs/SOURCE_SCOPED_JOB_SUMMARY.md @@ -41,6 +41,18 @@ reports and locating the underlying artifacts. A SHA-256 digest is tamper evidence only relative to a trusted expected digest; it does not establish the authenticity of the evidence or the GitHub runner. +**Portable human-review artifact (unreleased draft #228):** The same validated +text is also saved as `ovk-advisory-pr-summary.md` and included in the +existing OVK evidence-artifact upload. A teammate can inspect the report with +its exact source/decision limitations without copying the generic +`ovk-pr-comment.md`, which lacks these prominent source-binding caveats. +The standalone Markdown report is *not* a cryptographic certificate, and the +upload does not itself authenticate the file or event. Failed source validation +deletes any stale portable report before an always-running upload occurs. +The renderer deliberately excludes raw PR diffs and source excerpts from its +Markdown; the recorded digest supports subsequent independent comparison. +Native GitHub job-summary presentation remains available in parallel. + ## Reuse across repositories and CI systems The renderer itself uses only Python's standard library: @@ -50,7 +62,8 @@ python scripts/render_source_bound_job_summary.py \ --evidence ovk-evidence.json \ --source-binding ovk-source-binding.json \ --diff .ovk-pr.diff \ - --output advisory-summary.md + --output advisory-summary.md \ + --portable-output ovk-advisory-pr-summary.md ``` The evidence, source receipt, and **exact binary diff bytes** must come From 56256edd260c4fde10d39bc43d0daeb9f164ef41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:04:31 -0700 Subject: [PATCH 103/104] security(action): never upload raw PR diff in portable advisory artifacts --- action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/action.yml b/action.yml index 45c36da9c..f0db41171 100644 --- a/action.yml +++ b/action.yml @@ -487,6 +487,5 @@ runs: ovk-attestation-envelope.json ${{ inputs.bundle-output-dir }}/** ovk-branch-metadata.json - .ovk-pr.diff ovk-source-binding.json .ovk-check-emitted From 837c931d059f74dcda06432d83f5bde759f53d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=A9o=20H=2E=20Petel?= <113530345+fraware@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:04:39 -0700 Subject: [PATCH 104/104] test(report): prohibit uploading unredacted PR diffs --- tests/test_source_scoped_job_summary.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_source_scoped_job_summary.py b/tests/test_source_scoped_job_summary.py index c862e03ec..673aa1d5f 100644 --- a/tests/test_source_scoped_job_summary.py +++ b/tests/test_source_scoped_job_summary.py @@ -264,5 +264,8 @@ def test_action_uploads_scoped_report_without_uploading_raw_diff(tmp_path: Path) assert "inputs.verification-manifest == ''" in summary["if"] upload = next(s for s in steps if s.get("name") == "Upload OVK artifacts") assert "ovk-advisory-pr-summary.md" in upload["with"]["path"] + # A diff can contain credentials or private source. It is used and + # hashed locally, never included in portable uploaded artifacts. + assert ".ovk-pr.diff" not in upload["with"]["path"] cleanup = next(s for s in steps if s.get("name") == "Invalidate stale generated Action outputs") assert "ovk-advisory-pr-summary.md" in cleanup["run"]