diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc2349af1..d79e20523 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,27 +188,28 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - checks: write steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} + # GitHub's pull_request github.sha is the synthetic merge preview. + ref: ${{ github.sha }} + fetch-depth: 2 persist-credentials: false - - name: Strict emit-check on known-bad diff + - name: Strict Action must reject caller-selected unbound input id: ovk continue-on-error: true uses: ./ with: mode: strict use-check: "true" - emit-check: "true" + emit-check: "false" changed-files: examples/ci_secrets/workflow_secrets_on_pr.diff post-comment: "false" - - name: Assert strict block and observable check + - name: Assert unsafe input refused before verification run: | test "${{ steps.ovk.outcome }}" = "failure" - test "${{ steps.ovk.outputs.recommendation }}" = "block" - test "${{ steps.ovk.outputs.exit_code }}" = "1" + test ! -f ovk-source-binding.json + test ! -f ovk-evidence.json action_default_diff_dogfood: runs-on: ubuntu-latest @@ -217,7 +218,8 @@ jobs: steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} + # Match the actual GitHub event checkout subject, not PR-head. + ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false - name: Exercise automatic diff collection @@ -231,6 +233,20 @@ jobs: run: | test -f ovk-evidence.json test -f ovk-evidence-quality.json + test -f ovk-source-binding.json + python - <<'PY' + import json + import os + from pathlib import Path + r = json.loads(Path("ovk-source-binding.json").read_text()) + assert r["evaluated_checkout_commit_sha"] == os.environ["GITHUB_SHA"] + assert r["evidence_subject_sha"] == os.environ["GITHUB_SHA"] + if os.environ["GITHUB_EVENT_NAME"] == "pull_request": + assert r["input_comparison"] == "event_base_to_verified_merge_preview" + assert r["input_structurally_bound_to_checkout"] is True + assert r["bound_into_signed_evidence_envelope"] is False + assert r["production_reliance_authorized"] is False + PY ovk evidence-quality ovk-evidence.json package: diff --git a/LICENSE b/LICENSE index 724b7b640..d64569567 100644 --- a/LICENSE +++ b/LICENSE @@ -1,11 +1,202 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ -Copyright 2026 fraware + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION -http://www.apache.org/licenses/LICENSE-2.0 + 1. Definitions. -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in index 8979dc07b..5c932de78 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include README.md include LICENSE +include NOTICE recursive-include schemas *.json recursive-include templates *.intent.json recursive-include adapters *.json diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..7a3e3a963 --- /dev/null +++ b/NOTICE @@ -0,0 +1,5 @@ +Open Verification Kernel + +Copyright 2026 fraware + +Licensed under the Apache License, Version 2.0. See the LICENSE file for the complete terms. diff --git a/README.md b/README.md index ddffa0027..4dd40196f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](pyproject.toml) [![FormalPR-Bench](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/fraware/open-verification-kernel/main/docs/benchmarks/leaderboard-badge.json)](docs/benchmarks/latest-leaderboard-summary.json) -[![Docs](https://img.shields.io/badge/docs-v1.3.0--rc.1-green.svg)](docs/README.md) +[![Docs](https://img.shields.io/badge/docs-development--candidate-informational.svg)](docs/README.md) [Quick start](#quick-start) · [Add to CI](#github-actions) · [Contribute](#contribute) · [Documentation](docs/README.md) @@ -95,10 +95,12 @@ ovk repair-suggest --evidence ovk-evidence.json --- -## Quick start +## Quick start (local development snapshot) **Requirements:** Python 3.10+ +These commands clone current, unreleased development `main`. They are for local inspection and experimentation; they do **not** install an attributable published release or qualify a strict production gate. + ```bash git clone https://github.com/fraware/open-verification-kernel.git cd open-verification-kernel @@ -129,7 +131,7 @@ Installation, package, optional-solver, and signing paths: [docs/INTEGRATION.md] ## GitHub Actions -Add OVK to a pull request workflow. Start in **advisory** mode (reports findings without failing the job); move to **strict** only for lanes/profiles whose signal and trusted inputs you have calibrated for that repository. +Add OVK to a pull request workflow in **advisory** mode, with the repository maintainer's consent and no merge-blocking authority. The current public source profiles lack independent strict qualification; strict enforcement is a separate release, evidence, and source-model security decision. ```yaml name: Verify PR @@ -139,25 +141,30 @@ on: permissions: contents: read - pull-requests: write - checks: write # required when emit-check: true jobs: ovk: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 2 + persist-credentials: false + # SHA of the latest published v1.2.1 release's direct source commit. + # Change this only after reviewing the release identity and limits. + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 + env: + OVK_PACKAGE_VERSION: "1.2.1" with: - mode: advisory # switch to strict only after calibration - use-check: "true" # analyze the PR diff automatically - emit-check: "true" # optional; requires checks: write - post-comment: "true" # requires pull-requests: write + mode: advisory # reviewed, non-blocking compatibility pilot + use-check: "true" # legacy automatic PR-diff path; not a signed receipt + emit-check: "false" # no check-write authority + post-comment: "false" # no comment-write authority ``` -Pin target is `v1.3.0-rc.1` only after the attributable tag/release exists. Until then, do not treat that tag as a live install surface; see [docs/CURRENT_RELEASE_STATUS.md](docs/CURRENT_RELEASE_STATUS.md). +**Release safety boundary (verified 2026-10-15):** `v1.3.0-rc.1` is not published or tagged. This example runs **historical `v1.2.1` code**, pinned to its original source commit; it does not include the draft guarantee-continuity feature or the new fail-closed installer. **The `v1.2.1` Action falls back to its checkout if the requested PyPI version fails to install.** Do not treat that installation as independently authenticated solely because `OVK_PACKAGE_VERSION` is set. The old Git tag is unsigned, and its GitHub Release is not marked immutable. The corrected installation contract in draft PR #220 takes effect only in a later, independently authorized release. Until then use the checkout examples for development or a separately governed, consenting **advisory-only** experiment. See [release status](docs/CURRENT_RELEASE_STATUS.md), [pilot playbook](docs/EXTERNAL_PILOT_PLAYBOOK.md), and [launch checklist](docs/DEVELOPER_PREVIEW_READINESS.md). -Copy a full consumer example: [`examples/github_workflows/external_consumer.yml`](examples/github_workflows/external_consumer.yml) +Copy the **minimal advisory-only** consumer example: [`examples/github_workflows/external_consumer.yml`](examples/github_workflows/external_consumer.yml). It uses the historical Action and discloses its fallback limitation; it does not enable strict enforcement or require write tokens. --- diff --git a/SECURITY.md b/SECURITY.md index c0a239255..ee6a1a6b5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,19 +2,22 @@ OVK is security-sensitive infrastructure because it may influence pull-request merge decisions and evidence claims. -## Supported versions +## Release and security support status -| Version | Supported | +**Last reviewed:** 2026-10-15. The latest publicly published release is `v1.2.1`. The `v1.3.0-rc.1` tag and distribution are **not published**. A historical version appearing in this table does not establish that it receives backported security fixes. + +| Version | Public release and support status | |---|---| -| 1.1.x | yes | -| 1.0.x | yes | -| < 1.0 | no | +| `1.2.1` | Latest published release; the maintainer must explicitly confirm the security-patch support commitment | +| `1.1.x`, `1.0.x` | Historical versions; earlier documentation marked these supported, but ongoing security maintenance is not independently verified | +| Before `1.0` | No support claim | +| `1.3.0-rc.1` | Unpublished engineering candidate; no public install or security-support claim | -Report security issues against `main` or the latest release tag. +Report suspected vulnerabilities affecting the published release or current source tree through a **private** channel. Do not assume that reporting a problem against an old series creates a backport commitment. ## Reporting a vulnerability -Please open a private security advisory if available, or contact the maintainers through the repository owner. +Use the repository's private vulnerability-reporting / security advisory interface if available; otherwise contact the repository maintainer privately before sharing exploit details. If no private reporting path is available, the maintainers must configure one before a wider security-sensitive launch. ## Security-critical project rules diff --git a/action.yml b/action.yml index 0bcf30906..f0db41171 100644 --- a/action.yml +++ b/action.yml @@ -83,6 +83,19 @@ outputs: runs: using: composite steps: + - name: Invalidate stale generated Action outputs + shell: bash + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + # Generated OVK filenames are reserved Action outputs, not inputs. + # Upload runs with if: always(); stale files must never leak through. + rm -f -- .ovk-effective-input-path .ovk-pr.diff .ovk-action-exit-code + rm -f -- .ovk-emit-exit-code .ovk-check-emitted ovk-source-binding.json + rm -f -- ovk-evidence.json ovk-pr-comment.md ovk-attestation.json + rm -f -- ovk-artifact-manifest.json ovk-evidence-quality.json + rm -f -- ovk-provenance.json ovk-attestation-envelope.json + rm -f -- ovk-advisory-pr-summary.md - name: Cache pip packages uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: @@ -103,8 +116,14 @@ runs: pip install . } if [ -n "${OVK_PACKAGE_VERSION:-}" ]; then - pip install "open-verification-kernel==${OVK_PACKAGE_VERSION}" || install_from_checkout + # A requested distribution version is an identity constraint. + # Never silently substitute the Action checkout if PyPI resolution fails. + echo "OVK install: requested distribution version ${OVK_PACKAGE_VERSION}" + pip install "open-verification-kernel==${OVK_PACKAGE_VERSION}" + python -c 'from importlib.metadata import version; print("OVK installed distribution version:", version("open-verification-kernel"))' else + # Action source installs remain available for SHA-pinned development. + echo "OVK install: package from Action source checkout" install_from_checkout fi - name: Initialize verification directory if needed @@ -125,40 +144,31 @@ runs: shell: bash working-directory: ${{ github.workspace }} env: + OVK_EVENT_NAME: ${{ github.event_name }} OVK_CHANGED_FILES: ${{ inputs.changed-files }} + OVK_MODE: ${{ inputs.mode }} OVK_EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} OVK_EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} OVK_GITHUB_SHA: ${{ github.sha }} run: | set -euo pipefail - if [ -n "$OVK_CHANGED_FILES" ]; then - if [ ! -f "$OVK_CHANGED_FILES" ]; then - echo "OVK changed-files input does not exist: $OVK_CHANGED_FILES" >&2 - exit 1 - fi - printf '%s\n' "$OVK_CHANGED_FILES" > .ovk-effective-input-path - exit 0 - fi - - HEAD_SHA="${OVK_EVENT_HEAD_SHA:-$OVK_GITHUB_SHA}" - BASE_SHA="${OVK_EVENT_BASE_SHA:-}" - if [ -n "$BASE_SHA" ] && ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then - git fetch --no-tags --depth=1 origin "$BASE_SHA" || true - fi - if [ -n "$HEAD_SHA" ] && ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then - git fetch --no-tags --depth=1 origin "$HEAD_SHA" || true - fi - if [ -n "$BASE_SHA" ] && git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null \ - && [ -n "$HEAD_SHA" ] && git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then - git diff --no-ext-diff --binary "$BASE_SHA" "$HEAD_SHA" > .ovk-pr.diff - elif [ -n "$BASE_SHA" ] && git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then - git diff --no-ext-diff --binary "$BASE_SHA" HEAD > .ovk-pr.diff - elif git rev-parse HEAD^ >/dev/null 2>&1; then - git diff --no-ext-diff --binary HEAD^ HEAD > .ovk-pr.diff - else - git show --format= --no-ext-diff --binary HEAD > .ovk-pr.diff - fi - printf '%s\n' ".ovk-pr.diff" > .ovk-effective-input-path + python "${{ github.action_path }}/scripts/prepare_verified_change_input.py" \ + --event-name "$OVK_EVENT_NAME" \ + --github-sha "$OVK_GITHUB_SHA" \ + --base-sha "$OVK_EVENT_BASE_SHA" \ + --pr-head-sha "$OVK_EVENT_HEAD_SHA" \ + --changed-files "$OVK_CHANGED_FILES" \ + --mode "$OVK_MODE" + - name: Disclose unbound caller-selected inputs + if: ${{ inputs.changed-files != '' && inputs.verification-manifest == '' }} + shell: bash + run: | + echo '::warning::OVK caller-supplied change input is not bound to the checked-out source; automated GitHub Check/comment emission is disabled.' + - name: Disclose unbound manifest inputs + if: ${{ inputs.verification-manifest != '' }} + shell: bash + run: | + echo '::warning::OVK multi-lane manifest inputs lack source-to-PR binding; only local advisory artifacts are permitted. Automatic Check/comment publication and strict enforcement are disabled.' - name: Run OVK multi-lane verification id: verify if: ${{ inputs.verification-manifest != '' }} @@ -171,6 +181,27 @@ runs: OVK_REPO: ${{ github.repository }} OVK_HEAD_SHA: ${{ github.sha }} run: | + set -euo pipefail + # This path bypasses PR source preparation. Remove source-bound + # and previously generated observations left by any earlier Action + # invocation in the same workspace. No stale result is portable. + rm -f -- .ovk-effective-input-path .ovk-pr.diff ovk-source-binding.json + rm -f -- .ovk-check-emitted .ovk-emit-exit-code .ovk-action-exit-code + rm -f -- ovk-evidence.json ovk-pr-comment.md ovk-attestation.json + rm -f -- ovk-artifact-manifest.json ovk-evidence-quality.json + rm -f -- ovk-provenance.json ovk-attestation-envelope.json + # Never append new evidence to a directory with old bundle material. + # Caller-selected output roots are not deleted by the Action. + if [ -e "$OVK_BUNDLE_OUTPUT_DIR" ]; then + if [ ! -d "$OVK_BUNDLE_OUTPUT_DIR" ] || [ -n "$(find "$OVK_BUNDLE_OUTPUT_DIR" -mindepth 1 -print -quit)" ]; then + echo '::error::OVK manifest output directory already contains files; use an empty distinct directory.' + exit 2 + fi + fi + if [ "$OVK_MODE" != "advisory" ]; then + echo '::error::OVK manifest verification lacks PR source binding; strict Action enforcement is refused.' + exit 2 + fi args=( --manifest "$OVK_VERIFICATION_MANIFEST" --output-dir "$OVK_BUNDLE_OUTPUT_DIR" @@ -200,6 +231,8 @@ runs: OVK_MODE: ${{ inputs.mode }} OVK_CHECK_METADATA: ${{ inputs.check-metadata }} OVK_GITHUB_EVENT: ${{ github.event_path }} + OVK_EVENT_NAME: ${{ github.event_name }} + OVK_CHANGED_FILES: ${{ inputs.changed-files }} OVK_REPO: ${{ github.repository }} OVK_HEAD_SHA: ${{ github.sha }} run: | @@ -211,6 +244,12 @@ runs: --head-sha "$OVK_HEAD_SHA" --output-dir . ) + if [ "$OVK_EVENT_NAME" = "pull_request" ] && [ -z "$OVK_CHANGED_FILES" ]; then + # The locally generated source receipt was checked against the + # checkout and event. Bind the evidence subject to that exact merge, + # not to the contributor PR head re-read from the event payload. + args+=(--source-binding ovk-source-binding.json) + fi if [ -n "$OVK_CHECK_METADATA" ]; then args+=(--check-metadata "$OVK_CHECK_METADATA") elif [ -f ovk-branch-metadata.json ]; then @@ -231,46 +270,32 @@ runs: fi set -e echo "$ACTION_RC" > .ovk-action-exit-code - - name: Emit OVK GitHub check (check path) - id: emit-check-run - if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest == '' && inputs.use-check == 'true' }} + - name: Share scoped advisory observation in job summary + if: ${{ github.event_name == 'pull_request' && inputs.mode == 'advisory' && inputs.verification-manifest == '' && inputs.changed-files == '' && inputs.use-check == 'true' }} shell: bash working-directory: ${{ github.workspace }} - env: - GITHUB_TOKEN: ${{ github.token }} - OVK_REPO: ${{ github.repository }} - OVK_HEAD_SHA: ${{ github.sha }} run: | - set +e - python "${{ github.action_path }}/scripts/emit_github_check.py" \ + set -euo pipefail + python "${{ github.action_path }}/scripts/render_source_bound_job_summary.py" \ --evidence ovk-evidence.json \ - --markdown ovk-pr-comment.md \ - --repo "$OVK_REPO" \ - --head-sha "$OVK_HEAD_SHA" - EMIT_RC=$? - set -e - if [ "$EMIT_RC" -eq 0 ]; then - echo "true" > .ovk-check-emitted - else - echo "$EMIT_RC" > .ovk-emit-exit-code - fi - - name: Emit OVK GitHub check (verify path) - id: emit-check-verify - if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest != '' }} + --source-binding ovk-source-binding.json \ + --diff .ovk-pr.diff \ + --output "$GITHUB_STEP_SUMMARY" \ + --portable-output ovk-advisory-pr-summary.md + - name: Emit OVK GitHub check (check path) + id: emit-check-run + if: ${{ inputs.emit-check == 'true' && inputs.verification-manifest == '' && inputs.use-check == 'true' && inputs.changed-files == '' }} shell: bash working-directory: ${{ github.workspace }} env: GITHUB_TOKEN: ${{ github.token }} - OVK_BUNDLE_OUTPUT_DIR: ${{ inputs.bundle-output-dir }} OVK_REPO: ${{ github.repository }} OVK_HEAD_SHA: ${{ github.sha }} run: | - EVIDENCE="$OVK_BUNDLE_OUTPUT_DIR/ovk-evidence.json" - MARKDOWN="$OVK_BUNDLE_OUTPUT_DIR/ovk-pr-comment.md" set +e python "${{ github.action_path }}/scripts/emit_github_check.py" \ - --evidence "$EVIDENCE" \ - --markdown "$MARKDOWN" \ + --evidence ovk-evidence.json \ + --markdown ovk-pr-comment.md \ --repo "$OVK_REPO" \ --head-sha "$OVK_HEAD_SHA" EMIT_RC=$? @@ -329,7 +354,7 @@ runs: set -e echo "$ACTION_RC" > .ovk-action-exit-code - name: Post OVK pull-request comment - if: ${{ inputs.post-comment == 'true' }} + if: ${{ inputs.post-comment == 'true' && inputs.changed-files == '' && inputs.verification-manifest == '' }} shell: bash working-directory: ${{ github.workspace }} env: @@ -353,6 +378,7 @@ runs: working-directory: ${{ github.workspace }} env: OVK_VERIFICATION_MANIFEST: ${{ inputs.verification-manifest }} + OVK_CHANGED_FILES: ${{ inputs.changed-files }} OVK_BUNDLE_OUTPUT_DIR: ${{ inputs.bundle-output-dir }} OVK_EVIDENCE_OUTPUT: ${{ inputs.evidence-output }} run: | @@ -411,6 +437,14 @@ runs: } exit_code = str(exit_codes.get(decision_state or recommendation, 2)) + if os.environ.get("OVK_VERIFICATION_MANIFEST") or os.environ.get("OVK_CHANGED_FILES"): + # Caller-selected manifests and diffs are not source-bound PR + # inputs. Their scoped claim results stay in local evidence only; + # never export PASS as an Action-wide downstream allow gate. + decision_state = "unknown" + recommendation = "require_human_review" + exit_code = "2" + check_emitted = "true" if Path(".ovk-check-emitted").exists() else "false" github_output = os.environ["GITHUB_OUTPUT"] with open(github_output, "a", encoding="utf-8") as handle: @@ -445,6 +479,7 @@ runs: path: | ovk-evidence.json ovk-pr-comment.md + ovk-advisory-pr-summary.md ovk-attestation.json ovk-artifact-manifest.json ovk-evidence-quality.json @@ -452,5 +487,5 @@ runs: ovk-attestation-envelope.json ${{ inputs.bundle-output-dir }}/** ovk-branch-metadata.json - .ovk-pr.diff + ovk-source-binding.json .ovk-check-emitted diff --git a/docs/CURRENT_RELEASE_STATUS.md b/docs/CURRENT_RELEASE_STATUS.md index 3f71a3173..cffcad2ab 100644 --- a/docs/CURRENT_RELEASE_STATUS.md +++ b/docs/CURRENT_RELEASE_STATUS.md @@ -2,9 +2,11 @@ Living release/adoption status for Open Verification Kernel. -**Last reviewed:** 2026-08-26 +**Release/ref state checked live:** 2026-10-15 (other program and maturity claims remain subject to their separate evidence). -**Current judgment:** `1.3.0-rc.1` is an **engineering release candidate after the merge of final technical-closure PR #23**. It is not yet an attributable `v1.3.0-rc.1` release. The last previously signed immutable release remains `v1.2.1`; evidence from that tag must not be re-attributed to the RC source tree. The exact RC source identity is the final post-documentation candidate SHA that passes the complete exact-head development matrix; later movement of `main` does not redefine or authorize that candidate. +**Current judgment:** `1.3.0-rc.1` is an **unpublished engineering release candidate**, not an attributable `v1.3.0-rc.1` release. On 2026-10-15, the RC tag lookup returned 404; the latest GitHub Release is `v1.2.1` (published 2026-07-23). Its annotated Git tag is reported by GitHub as **unsigned** (`verification.verified=false`, `reason=unsigned`), and its GitHub Release object reports `immutable=false`. Wheel/sdist Sigstore bundle assets are attached, but their cryptographic validity was **not independently verified by this live status review**. Do not conflate distribution signing with Git-tag signing or release immutability. + +`main` resolved to `79370539ca9b26a0a6b01f6ac6ab6923243f40f4` and reported `protected=false`; the repository rulesets list was empty. The branch-protection-detail endpoint was inaccessible to this integration (403), so administrative configuration was not inspected exhaustively. These observations do **not** establish the necessary release-ref protections. Issue #25 remains blocking. Historical artifacts from `v1.2.1` must never be re-attributed to an RC source tree. A new candidate requires the complete exact-head development matrix and independently authorized tag-bound release evidence. The authoritative publication procedure is [RELEASE.md](RELEASE.md). The evidence threshold is [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). The generated TCB is [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md). @@ -124,7 +126,7 @@ Safe recovery after a partial PyPI/GitHub failure is allowed only when existing | Strict required check | Repository-specific only after trusted inputs and empirical calibration for the intended lane | | General production-stable enforcement | **Not claimed** | -Until an attributable `v1.3.0-rc.1` release exists, do not present that tag as a live install pin. Historical `v1.2.1` remains a separate immutable release with its own source and signing evidence. +Until an attributable `v1.3.0-rc.1` release exists, do not present that tag as a live install pin. Historical `v1.2.1` is a separate **published release**, with attached signing-related assets; its **Git tag is unsigned and GitHub Release immutability is false** according to the October 15, 2026 API observation. Its distribution bundles have not been cryptographically reverified by this review. ## Remaining external/maintainer evidence for the RC diff --git a/docs/DEVELOPER_PREVIEW_READINESS.md b/docs/DEVELOPER_PREVIEW_READINESS.md new file mode 100644 index 000000000..b78f0c5b2 --- /dev/null +++ b/docs/DEVELOPER_PREVIEW_READINESS.md @@ -0,0 +1,59 @@ +# Developer preview readiness and first-user activation + +**Status: NOT LAUNCH-AUTHORIZED.** This checklist is for the maintainer and an independent release reviewer. Do not present green PR CI or a draft PR as a published, signed production version. + +## The promise to test + +For one supported change in a real PR, the developer receives a reviewable machine-readable result that identifies (a) the bounded claim checked, (b) what evidence supports the outcome and the relevant source revision, (c) unsupported/unknown conditions, and (d) the exact next verification action. The objective is to make security-sensitive AI-authored changes reviewable with accountable evidence. Do not call an entire repository or PR formally verified. + +**Distribution strategy:** open-source CLI and composite GitHub Action as the first developer-native interface; shareable PR reports once their source revision and trust scope are reliably bound; GitHub App and optional organization service only following repeated external demand. A standalone hosted web application is not a release prerequisite. + +## First local demonstration (unreleased checkout only) + +The following runs against the **source checkout**, not a published distribution. Use Python 3.10–3.12 in a clean environment: + +```bash +git clone https://github.com/fraware/open-verification-kernel.git +cd open-verification-kernel +python -m pip install -e '.[dev]' +ovk init +ovk doctor +ovk check --changed-files examples/multi_surface/pr_combined.diff --advisory +``` + +Inspect the generated `ovk-evidence.json`, `ovk-pr-comment.md`, and `ovk-evidence-quality.json`. These are **bounded development fixtures** for product understanding, not independent accuracy evidence. Try the deterministic failing/repair walkthrough with: + +```bash +python examples/repair_loops/ci_secrets/demo_repair_loop.py +``` + +The emitted outputs must be explained in the context of the admitted inputs and compiler/checker assumptions. In advisory mode the job can succeed while the **recorded decision** remains `block`, `needs_review`, or `unknown`. + +## Historical release versus current development + +As checked October 15, 2026, `v1.2.1` is the last public GitHub Release; `v1.3.0-rc.1` **does not exist as a public tag**. The historical source commit for `v1.2.1` is `a27d5720f4350c00bca34f71d991c31f5a2f38c7`, whose **old Action implementation silently falls back from failed versioned PyPI installation to its checkout**. Its annotated tag is unsigned and the Release object is not marked immutable. This is insufficient to claim an authenticated, fail-closed distribution solely from the Action reference. + +Draft [PR #220](https://github.com/fraware/open-verification-kernel/pull/220) corrects the current checkout's installer; it does **not** retroactively patch the old Action. Do not advertise a corrected release until it is independently authorized and published. + +## Release GO / NO-GO acceptance criteria + +- [ ] GitHub administrator creates active branch and tag rulesets and the effective constraints are independently read back. See [issue #25](https://github.com/fraware/open-verification-kernel/issues/25). +- [ ] The complete exact SHA candidate passes CI, multi-OS/Python reproducibility, native backend matrices, package build, outside-checkout smoke, templates, and Action dogfood. +- [ ] The requested distribution version either installs exactly or the installation fails, with no silent checkout fallback; negative test runs on each supported platform. +- [ ] Final README, example workflows, security support policy, attribution, and license are consistent with the publication state. No nonexistent tag is in **active** copyable examples. +- [ ] The signed annotated release tag directly identifies the authorized source SHA and GitHub reports verified signature; new tag-bound release, holdout, consumer, and ledger checks meet [RELEASE.md](RELEASE.md). +- [ ] Sigstore material is cryptographically verified against exact published distribution hashes, and PyPI read-back equals the authorized bytes. Report historical artifact-signing and tag-signing claims separately. +- [ ] One external maintainer explicitly consents to a **nonblocking advisory** pilot with their normal PR stream. A reviewer inspects at least one real supported/unsupported change, recording denominators, unknowns and review time. Small samples cannot qualify strict enforcement. +- [ ] A new developer outside the maintainer team completes the **source checkout / released-wheel** first-run task and explains one result, without private fixture knowledge or maintainer intervention. Record errors and elapsed setup time. + +All boxes are independent; the absence of any source/provenance or policy evidence prohibits broad claims. The presence of published code does not grant automatic merge authority. + +## Adoption and repeat-use metrics + +Measure the conversion sequence **README viewed → independently installed → first meaningful report → second natural PR reviewed → teammate adoption**. Exclude bots and the maintainer's own controlled repositories from independent organization counts. Track unsupported source coverage as prominently as supported paths. Record reviewer time and avoided rechecking separately from compute duration; do not infer a breach-prevention rate from a tiny sample. + +**Primary North Star metric:** independently reviewed, consequential software guarantees that remain justified across successive changes, with observed revalidation effort and explicit unknowns. Stars, website impressions, and reposts are secondary distribution signals, not proof of customer value. + +## Expansion decision + +A public GitHub App follows repeated Action use **and** demonstrated onboarding friction that an App meaningfully reduces. A hosted evidence history follows paid demand for multi-repository persistence, access control, revocation, audit policy, and operational support. Public viral marketing follows a real consented demonstration with exact version-pinned, replayable materials and carefully stated guarantees. diff --git a/docs/EXTERNAL_PILOT_PLAYBOOK.md b/docs/EXTERNAL_PILOT_PLAYBOOK.md index 6f5af993a..a01bce36c 100644 --- a/docs/EXTERNAL_PILOT_PLAYBOOK.md +++ b/docs/EXTERNAL_PILOT_PLAYBOOK.md @@ -1,146 +1,127 @@ -# External OSS Pilot Playbook +# External OSS pilot playbook — advisory evaluation -Guide for adding OVK to an external open-source repository: start in advisory mode, measure false positives, then optionally enforce on protected branches. +This playbook governs **consented, non-blocking** use of Open Verification Kernel (OVK) in an independently operated repository. An external pilot is a test of customer utility and the declared verification profile, **not** proof of general source-level security, production readiness, or verified economic savings. -## Start with one check type +The current public status identifies source profiles as `executable_advisory`. No `externally_calibrated_strict` source-profile qualification is established. A small pilot must never automatically promote a check to strict or to a required merge gate. -| Check type | Best for | Starter manifest | -|------|----------|------------------| -| CI secrets | Repos with agent-authored workflow PRs | [pilot_manifest_ci_secrets.template.json](templates/pilot_manifest_ci_secrets.template.json) | -| Self-protection | Repos adding agent CI gates | `examples/pilot_repos/self_protection_only.json` | +## 0. Obtain consent and qualify the repository -Enable additional check types one at a time after each is stable on your diffs. +Before adding any GitHub workflow, obtain approval from the maintaining organization, identify a reviewer and a technically responsible operator, and agree on which evidence is safe to retain or publish. Do not install CI into someone else's repository without permission. -## Step 1 — Advisory mode (no merge blocking) +Record the target repository and exact PR revision, the declared check family, its supported source constructs and exclusions, the intended high-risk operation, the reviewer decision, and the organization's actual need. A changed authentication-related file is **not** sufficient proof that OVK supports the code's authorization semantics. Keep unsupported shapes **unknown** and subject to human review. -1. Copy `docs/templates/pilot_manifest_ci_secrets.template.json` to `.verification/ci_secrets_pilot.json` in the target repo. -2. Add check input JSON under `.verification/ci_secrets/` (see `examples/ci_secrets/input_secrets_safe.json` for shape). -3. Add OVK to CI in advisory mode. Build the PR diff locally — OVK reads files, not URLs: +Start with one available check family. The historical example manifests include: + +| Check family | Potential starting problem | Starter manifest | +| --- | --- | --- | +| CI secrets | Changes to workflow secret exposure | [CI secrets template](templates/pilot_manifest_ci_secrets.template.json) | +| Self-protection | Changes to agent CI gates | `examples/pilot_repos/self_protection_only.json` | + +The FastAPI authorization / Protected Effect path is a separate bounded advisory research profile. It must first pass source-eligibility and coverage screening; do not substitute the CI-secrets example for proof of route authorization. + +## 1. Install a released, attributable advisory build + +Use the [current release status](CURRENT_RELEASE_STATUS.md), [release procedure](RELEASE.md) and an exact vetted release/ref identity. The historic `v1.2.1` example below uses its **full 40-hex source commit**, not the unprotected tag: confirm the remote tag and distribution integrity for the installation, and never treat a draft `v1.3.0-rc.1` as a published release on the strength of green development CI. + +Copy `docs/templates/pilot_manifest_ci_secrets.template.json` into the consenting repository as `.verification/ci_secrets_pilot.json`, and replace sample inputs with real, explicitly in-scope paths. Do **not** treat `examples/ci_secrets/input_secrets_safe.json` as coverage of the adopter's code. + +The action should use least privileges and no automatic comments on untrusted fork PRs. Example starting point (adapt to the target repository's branch and trust settings): ```yaml -name: OVK Pilot +name: OVK advisory pilot on: pull_request: branches: [main] - -env: - OVK_PACKAGE_VERSION: "1.2.1" - +permissions: + contents: read jobs: ovk-advisory: runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 - - name: Build PR diff for OVK + persist-credentials: false + - name: Build PR diff run: | git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: OVK advisory check - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory use-check: "true" changed-files: ovk-pr.diff + emit-check: "false" post-comment: "false" - - name: OVK advisory verify (CI secrets check) - uses: fraware/open-verification-kernel@v1.2.1 + - name: OVK advisory verification + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json bundle-output-dir: ovk-pilot-bundle + emit-check: "false" post-comment: "false" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: always() + with: + name: ovk-pilot-artifacts + path: | + ovk-evidence.json + ovk-pilot-bundle/** + retention-days: 30 + if-no-files-found: warn ``` -4. Run for **two weeks** on all agent PRs without blocking merges. +Review the actual code and external actions before enabling this workflow. The full source-commit SHA pins the top-level Action, yet the historical Action may reference transitive dependencies by mutable tags and may fall back to installing the Action checkout if its requested distribution version is unavailable. Independently review that supply chain and keep the pilot non-blocking. The legacy caller-supplied diff is **unbound source input**, not an authenticated verified-PR receipt; the workflow explicitly disables both check publication and comments. The check stays advisory, cannot authorize a merge, and does not receive secrets from untrusted PRs. -## Step 2 — Measure +## 2. Observe all naturally occurring PRs -| Metric | Target before strict | -|--------|----------------------| -| False positive rate | under 5% | -| Time to first green after repair | under 5 minutes | -| Block rate on known-bad fixtures | 100% | +Begin with a 14-day window. Collect **all** natural PRs in the consented scope, not a favorable selection. A target of five PRs is useful to test deployment feasibility, **not** a safety or false-negative-rate threshold. Extend the observation window if activity is low. -Download the workflow artifact (`ovk-pilot-artifacts`) and ingest metrics into the OVK registry: - -```bash -python scripts/ingest_external_pilot_metrics.py --repo org/repo --artifacts-dir ./artifact -python scripts/render_pilot_metrics.py --registry docs/benchmarks/external-pilots-registry.json -``` +For each PR, record the exact base/head SHAs, source and verifier version, declared claim, supported/unsupported/unknown input, raw evidence and assumptions, report viewed by the reviewer, actual reviewer decision, adjudicated incorrect alerts, independently discovered misses, setup and review time, and fixes or rechecks. Record missing observations as missing—not as zeros. Keep private source and evidence out of the public registry unless explicitly approved. -Optional self-report JSON: copy [external_pilot_report.template.json](templates/external_pilot_report.template.json), fill measured values, and pass `--report external_pilot_report.json` to the ingest script. +If the pipeline reports a `PASS`, communicate the **typed checker/IR claim under its exact coverage assumptions**. Never summarize it as proof that an arbitrary source application is secure. If extraction is partial or an operation is outside the admitted profile, the outcome is unresolved. -Publish results in [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md) under **External pilots**. +## 3. Compute metrics with correct provenance -Maintainers ingest downloaded workflow artifacts and refresh the public summary: +The ingest command is: ```bash -python scripts/ingest_external_pilot_metrics.py --repo org/repo --artifacts-dir ./artifact -python scripts/render_pilot_metrics.py --registry docs/benchmarks/external-pilots-registry.json -``` - -Optional self-report JSON: copy [external_pilot_report.template.json](templates/external_pilot_report.template.json), fill metrics, and pass `--report external_pilot_report.json` to the ingest script. Registry rows live in [external-pilots-registry.json](benchmarks/external-pilots-registry.json). - -## Step 3 — Strict on protected branches - -When false positives stay below 5%: - -```yaml -- uses: fraware/open-verification-kernel@v1.2.1 - with: - mode: strict - use-check: "true" - emit-check: "true" +python scripts/ingest_external_pilot_metrics.py \ + --repo org/repo --artifacts-dir ./artifact \ + --report ./external_pilot_report.json +python scripts/render_pilot_metrics.py \ + --registry docs/benchmarks/external-pilots-registry.json ``` -Require the OVK check on `main` / `release/*` only; keep advisory on experimental branches if needed. - -## Step 4 — Add more check types +The optional report starts from [external_pilot_report.template.json](templates/external_pilot_report.template.json); replace its placeholders. Even if a file named `external_pilot_report.json` appears within a downloaded CI artifact directory, the ingester **does not automatically adopt its adjudication fields**. A human operator must deliberately select a separately reviewed report via `--report`, with the same repository identifier as `--repo`. Such reports remain self-reported until independently audited. `prs_evaluated`, `prs_blocked`, and latency can come from workflow artifacts. **False positives require separate human or independently specified ground-truth adjudication.** OVK's own `passed`, `merge_recommendation`, and checker `PASS/FAIL` fields are not ground-truth labels. The metric ingester must not manufacture a false-positive rate from them. -After CI secrets is stable, add check types one at a time using manifests from `examples/pilot_repos/`. Re-run advisory for each new check before strict enforcement. +Only supply `false_positives` and `false_positive_rate` if a reviewer actually classified cases and the denominator and decisions are documented in the report's linked evidence or notes. Otherwise leave both `null`. Every explicitly ingested artifact directory is treated as a potentially new, unreviewed cohort, including directories containing only one evidence bundle or no recognized aggregate. Previously reported error counts/rates are invalidated unless an explicit report supplies fresh adjudication. Cohort identity is not authenticated in this development ingester; operators must document report-to-artifact correspondence. Missing input must never be treated as zero observed errors. A reviewer-provided number is still **self-reported** until independently audited. -## Copy-paste workflow +Explicitly distinguish: +- **Check/recommendation rate:** what OVK returned on actual PRs. +- **Adjudicated false alarms:** the reviewer judged the tool's finding incorrect, with a defined denominator. +- **Independently found misses:** issues found by a separate source of truth that OVK did not flag, including unsupported cases. +- **Coverage/unknown fraction:** what proportion of natural PRs and relevant constructs fell outside the supported profile. +- **Economics:** reviewer minutes, CI latency/cost, integration effort and rechecks; compare with a credible baseline. -See `examples/github_workflows/pilot_fork_adopter.yml` and the full example in Step 1 above. Upload artifacts for your pilot report: +Small selected samples, even with zero observed false positives, cannot justify a general low-risk claim. -```yaml - - uses: actions/upload-artifact@v4 - with: - name: ovk-pilot-artifacts - path: | - ovk-evidence.json - ovk-pilot-bundle/** - retention-days: 30 -``` +## 4. Decision gate: continue, revise, or stop -## Reference in this repo +**Continue advisory:** the maintainer actively uses the findings and they remain appropriately scoped, with transparent unknowns and manageable review costs. -Weekly in-repo pilot workflow: `.github/workflows/pilot-dogfood.yml` with `scripts/collect_pilot_metrics.py`. +**Revise:** coverage misses important high-risk changes, the result is difficult to interpret, or false alarms dominate. Improve the *supported profile* or workflow only with evidence from actual users. -Published advisory reports (maintained consumers + infra profile): [pilots/README.md](pilots/README.md). +**Stop or select a different wedge:** the relevant guarantee is outside the verifier's modeled semantics, no reviewer values the output, or no willing maintainer participates. -## Support artifacts +**Strict enforcement is a distinct qualification project.** It needs a justified source-model correspondence, trustworthy evidence/provenance, independent profile-appropriate safety evaluation with explicit unknown handling, security review, a controlled rollback plan, repository-specific risk approval, and release/ref integrity. No numeric false-positive threshold or five-PR fixture test automatically grants strict maturity. -- Repair loop walkthrough: [AGENT_REPAIR_LOOP.md](AGENT_REPAIR_LOOP.md) -- Realistic PR diffs: `benchmarks/real_diffs/` -- Example workflows: `examples/github_workflows/` -- Metrics: `scripts/collect_pilot_metrics.py`, `docs/benchmarks/adoption-summary.json` -- External pilot registry: `docs/benchmarks/external-pilots-registry.json` -- Self-report template: `docs/templates/external_pilot_report.template.json` -- Ingest script: `scripts/ingest_external_pilot_metrics.py` +## 5. Reporting and references -## What to report +Publish a study only with maintainer permission and clearly label it as *independent external*, *maintained-consumer*, or *dogfood*. The existing [external pilot registry](benchmarks/external-pilots-registry.json) includes maintained and internal examples; they are not independent customer validation. -- Repository name (link) -- Check type(s) enabled -- Advisory duration (dates) -- PRs evaluated / blocked / false positives -- Median `ovk check` latency -- Strict mode enabled (yes/no) +The adoption report and supporting paths are [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md), `scripts/collect_pilot_metrics.py`, `scripts/ingest_external_pilot_metrics.py`, `scripts/render_pilot_metrics.py`, and `docs/benchmarks/external-pilots-registry.json`. -Add entries to [PILOT_CASE_STUDIES.md](PILOT_CASE_STUDIES.md). +Operational ownership and milestones are tracked in [North Star issue #216](https://github.com/fraware/open-verification-kernel/issues/216), with the first independent pilot in [issue #20](https://github.com/fraware/open-verification-kernel/issues/20). diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index c36591925..23922a9fd 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -24,12 +24,14 @@ pip install open-verification-kernel==1.3.0-rc.1 pip install "open-verification-kernel[solvers]==1.3.0-rc.1" ``` -Until the wheel is on PyPI, use `pip install -e '.[dev]'` from a checkout. **Live signed pin today:** `@v1.2.1` with `OVK_PACKAGE_VERSION: "1.2.1"`. **Candidate after attributable tag:** `@v1.3.0-rc.1` with matching `OVK_PACKAGE_VERSION`. Do not pin `@main` or treat uncommitted branch work as a verified release. +For current, **unreleased development code**, use `pip install -e '.[dev]'` from a checkout. The last published GitHub Release is `v1.2.1`; its annotated Git tag is unsigned, its GitHub Release is not marked immutable, and its historical Action **falls back to checkout code if the requested package version cannot be installed**. Its exact source commit is `a27d5720f4350c00bca34f71d991c31f5a2f38c7`. An attributable `v1.3.0-rc.1` release is **not currently available**. The commands in the preceding PyPI section are forward-looking and must not be run as current-release instructions. The fail-closed installer introduced in draft PR #220 will apply only to a later independently authorized release. Optional Z3: `pip install -e '.[solvers]'` Optional HMAC signing: `export OVK_SIGNING_KEY=your-signing-secret` +**Signing trust boundary:** `OVK_SIGNING_KEY` is a shared HMAC secret used for signing and verifying; it is not a public verification key. Do not expose it to fork-originated `pull_request` jobs, upload it with evidence artifacts, or mistake a locally HMAC-signed bundle for independently authenticated source or release provenance. + ## First demo Self-protection failing case (gate removed): @@ -91,8 +93,12 @@ permissions: **Third-party action pins:** `action.yml` and `.github/workflows/publish.yml` pin `actions/*` (and other release third-parties) to immutable commit SHAs. CI enforces this with `python scripts/pin_action_shas.py`. +**Historical example warning:** the Action snippets below use the exact `v1.2.1` source commit for reproducibility of the *GitHub source reference*, not an authorized new release. That Action still has an installation fallback. Their configurations are **advisory/legacy illustrations only**, not a claim of fully authenticated package-byte identity. Do not promote to a required strict check solely from this example. + ### Single check type (self-protection) +The following historical illustration includes optional check/comment publishing with write privileges. **Do not copy it as the first external pilot workflow.** For consenting adopters, use the [read-only advisory example](../examples/github_workflows/external_consumer.yml) and the [pilot playbook](EXTERNAL_PILOT_PLAYBOOK.md). + ```yaml name: OVK on: @@ -107,7 +113,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: fraware/open-verification-kernel@v1.2.1 + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 id: ovk with: mode: advisory @@ -122,10 +128,12 @@ jobs: Development reference: `uses: ./` +The recommended copyable external workflow is [the non-blocking, read-only advisory example](../examples/github_workflows/external_consumer.yml). Required strict checks are not qualified for the current source profiles. + ### All five check types (manifest) ```yaml -- uses: fraware/open-verification-kernel@v1.2.1 +- uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory verification-manifest: .verification/full_mvp.json @@ -161,15 +169,15 @@ When `verification-manifest` is set, the manifest path wins over `use-check`. Downstream example: ```yaml -- uses: fraware/open-verification-kernel@v1.2.1 +- uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 id: ovk with: - mode: strict + mode: advisory use-check: "true" emit-check: "true" -- name: Fail when OVK blocks +- name: Surface OVK block for human review if: steps.ovk.outputs.recommendation == 'block' - run: exit 1 + run: echo "OVK reported a block; inspect scope and supporting evidence" ``` **Artifact files:** @@ -217,7 +225,7 @@ Explicit metadata (most reliable): ``` ```yaml -- uses: fraware/open-verification-kernel@v1.2.1 +- uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: check-metadata: ovk-required-checks.json ``` @@ -252,16 +260,18 @@ Required GitHub check name: **`Open Verification Kernel`** (from `ovk/core/githu ### Rollout stages +**Stages 2–4 are future, separately qualified configurations**, not a current promotion path. Current public source profiles are advisory, the repository's protection and release-signing gates are open, and small in-repo samples do not authorize strict enforcement. + | Stage | Workflow example | Mode | Publish check run | Branch protection | |---|---|---|---|---| | 1 — Advisory only | `examples/github_workflows/pilot_fork_adopter.yml` | advisory | no | none | | 2 — Advisory + visibility | `examples/github_workflows/pilot_advisory_with_comment.yml` | advisory | yes | optional | -| 3 — Strict enforcement | `examples/github_workflows/external_consumer.yml` | strict | yes | required on `main` | +| 3 — Strict enforcement (future) | No currently approved copyable workflow | strict | qualified-only | approved protection | | 4 — Required check wired | `examples/github_workflows/branch_protection_required_check.yml` | strict | yes | required check | -Steps for strict enforcement: +Steps for strict enforcement **only following source-profile, release, independent adjudication, owner approval and rollback qualification**: -1. Run strict + `emit-check: true` on a sample PR and confirm the check appears on the commit. +1. On the separately authorized repository and exact source profile, test strict + `emit-check: true` and confirm correct source binding and check attribution. 2. In repository **Settings → Branches**, add **`Open Verification Kernel`** as a required status check on protected branches. 3. Supply explicit `check-metadata` when verifying gate preservation (auto-collected metadata is best-effort only). @@ -269,6 +279,8 @@ Steps for strict enforcement: ## External repository smoke test +The following is a **historical, write-enabled integration example**, not a copyable first-user or fork-safe installation. The default external pilot uses the read-only example in the playbook. + Create `.github/workflows/ovk-smoke.yml` in an integration repository: ```yaml @@ -285,7 +297,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: fraware/open-verification-kernel@v1.2.1 + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 env: OVK_PACKAGE_VERSION: "1.2.1" with: diff --git a/docs/PILOT_CASE_STUDIES.md b/docs/PILOT_CASE_STUDIES.md index 8f145eb1a..13fdd5fdd 100644 --- a/docs/PILOT_CASE_STUDIES.md +++ b/docs/PILOT_CASE_STUDIES.md @@ -2,7 +2,7 @@ Measured outcomes from example configurations in `examples/pilot_repos/`. Metrics use `ovk check` and `ovk verify` in advisory mode. -**Honesty:** in-repo pilot dogfood is **not** independent consumer validation and does not authorize `verified_source_sha` or `externally_calibrated_strict`. Live signed pin for external remotes remains `v1.2.1` until `v1.3.0-rc.1` is attributable. +**Honesty:** in-repo pilot dogfood is **not** independent consumer validation and does not authorize `verified_source_sha` or `externally_calibrated_strict`. Fixture pass/block output does not establish a false-positive rate; unadjudicated error rates are reported as unknown. The last published tag is `v1.2.1`, whose annotated Git tag was reported unsigned by GitHub on 2026-10-15; do not call it a Git-signed or protected pin. For any permitted external advisory experiment, use a reviewed **full 40-hex commit SHA** and independently verify applicable release assets and operator trust. ## In-repo reference pilot @@ -12,14 +12,14 @@ Simulates how an external repo would consume OVK before community pilots land. **Manifest:** `examples/pilot_repos/external_oss_ci_secrets.json` -**Consumer path:** local Action (`./`) with package version matching the workflow under test. Prefer mirroring `@v1.2.1` for live signed pins; bump only after an attributable RC tag. +**Consumer path:** local Action (`./`) with package version matching the workflow under test. Prefer the audited commit SHA `a27d5720f4350c00bca34f71d991c31f5a2f38c7` behind the published `v1.2.1` annotated tag for a legacy advisory Action example, with explicit owner review. This pins code identity but does not independently verify signer, source semantics, or dependency integrity. Bump only after an attributable new release. | Metric | Target | Measured | |--------|--------|----------| | Advisory mode | yes | yes | | Unsafe workflow diff block rate | 100% | _from weekly artifact_ | | Safe manifest pass rate | 100% | _from weekly artifact_ | -| False positive rate (fixtures) | 0% | _from pilot-metrics.json_ | +| Adjudicated false-positive rate | — | Unknown; the fixture outcomes contain no independent reviewer labels | | Median manifest latency | — | _from pilot-metrics.json_ | | Adoption summary | published | `docs/benchmarks/adoption-summary.json` | @@ -45,7 +45,7 @@ python scripts/render_pilot_metrics.py --metrics pilot-dogfood-report.json |--------|--------| | Time to first green run | ~0.3s | | Check types run | 1 | -| False positive rate (fixtures) | 0% (1/1 pass case) | +| Adjudicated false-positive rate | Unknown; one passing fixture is not independent error adjudication | Lowest-friction path for repos adding agent-authored PR gates. @@ -67,7 +67,7 @@ Lowest-friction path for repos adding agent-authored PR gates. |--------|--------| | Time to green (all pass fixtures) | ~2.1s | | Check types run | 5 | -| False positive rate (fixtures) | 0% (5/5 pass cases) | +| Adjudicated false-positive rate | Unknown; five passing fixtures are not independent error adjudication | | Artifacts | evidence, markdown, attestation, manifest, quality report | ## Cedar, TLA+, and Kani backends @@ -124,12 +124,12 @@ Source of truth: [external-pilots-registry.json](benchmarks/external-pilots-regi | Repository | Status | Kind | Check types | Advisory period | False positive rate | Strict enabled | |-------|--------|------|-------------|-----------------|---------------------|----------------| -| [fraware/ovk-consumer-fastapi-terraform](pilots/fastapi-terraform/REPORT.md) | advisory | Maintained consumer | ci_secrets, infrastructure | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | -| [fraware/ovk-consumer-express-actions](pilots/express-actions/REPORT.md) | advisory | Maintained consumer | ci_secrets, self_protection | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | -| [in-repo/ovk-pilot-infra-terraform-k8s](pilots/infra-terraform-k8s/REPORT.md) | advisory | In-repo maintained profile | infrastructure, ci_secrets | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | +| [fraware/ovk-consumer-fastapi-terraform](pilots/fastapi-terraform/REPORT.md) | advisory | Maintained consumer | ci_secrets, infrastructure | 2026-07-11 – 2026-07-25 | unknown (unadjudicated fixtures) | no | +| [fraware/ovk-consumer-express-actions](pilots/express-actions/REPORT.md) | advisory | Maintained consumer | ci_secrets, self_protection | 2026-07-11 – 2026-07-25 | unknown (unadjudicated fixtures) | no | +| [in-repo/ovk-pilot-infra-terraform-k8s](pilots/infra-terraform-k8s/REPORT.md) | advisory | In-repo maintained profile | infrastructure, ci_secrets | 2026-07-11 – 2026-07-25 | unknown (unadjudicated fixtures) | no | | TBD - recruiting first true external OSS adopter | recruiting | True external OSS (open) | ci_secrets | — | — | no | -Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurements with `strict_mode_recommendation: remain_advisory`. They do not replace a true independent external OSS adopter. When such a repo completes advisory rollout, ingest artifacts, update the registry, re-render the adoption summary, and keep the kind label explicit. Target: under 5% false positives before enabling strict mode on protected branches. +Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurements with `strict_mode_recommendation: remain_advisory`. They do not replace a true independent external OSS adopter. When such a repo completes advisory rollout, ingest artifacts, update the registry, re-render the adoption summary, and keep the kind label explicit. No numeric fixture-rate threshold authorizes strict enforcement. The formerly displayed 0.0 error rates were not independently adjudicated and are now null in the registry and adoption summary; historical values remain in Git history. An independent pilot needs explicit, reviewed false-positive and false-negative denominators, coverage and unknown handling. ### Reporting template @@ -137,14 +137,14 @@ Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurement **Manifest:** adapted from `examples/pilot_repos/external_oss_ci_secrets.json` or `self_protection_only.json` -**Rollout:** advisory (at least 14 days) → strict on `main` +**Rollout:** advisory (at least 14 days) → independent review of qualified source coverage, sample adequacy, unknowns, false negatives, false positives, governance and rollback → optional separately approved strict deployment. There is no automatic promotion based on elapsed time or one metric. | Metric | Target | Measured | |--------|--------|----------| | Advisory period | ≥14 days | _TBD_ | | PRs scanned | — | _TBD_ | | Block rate on unsafe workflow diff | 100% | _TBD_ | -| False positive rate | <5% before strict | _TBD_ | +| False-positive numerator/denominator and uncertainty | report all cases; no automatic strict threshold from small N | _TBD_ | | Time to first green after repair | <5 minutes | _TBD_ | | Strict enable date | — | _TBD_ | | p95 Action latency | — | _TBD_ | @@ -153,6 +153,6 @@ Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurement ### Publication checklist - [ ] Advisory window ≥14 days with artifact retention -- [ ] False-positive rate computed and under 5% (or strict not yet enabled) -- [ ] Link to workflow file and version pin (`@v1.2.1` live; `@v1.3.0-rc.1` after attributable cut) +- [ ] Report false positives with denominator and sampling limitations; strict remains off until independent source-profile, sensitivity/false-negative and policy qualification +- [ ] Link to workflow file and exact immutable commit SHA; do not rely on unprotected release tags as immutable installation pins - [ ] Update the **Active and recruiting** table with measured values diff --git a/docs/PR_SOURCE_BINDING_V1.md b/docs/PR_SOURCE_BINDING_V1.md new file mode 100644 index 000000000..0fba144cc --- /dev/null +++ b/docs/PR_SOURCE_BINDING_V1.md @@ -0,0 +1,72 @@ +# PR source identity contract — merge-preview development v1 + +**Security and release prerequisite; non-controlling.** This change addresses [#222](https://github.com/fraware/open-verification-kernel/issues/222). It makes the Action's *default PR diff* refer to the same Git commit as the existing `--head-sha` evidence subject. It is **not** a complete authenticated verification receipt or a proof of source-to-model fidelity. + +## Distinct objects + +For a GitHub `pull_request`, the event reports `base.sha` and `head.sha`. The Actions `github.sha` normally identifies a GitHub-generated merge-preview commit, not the contributor head. `actions/checkout` normally checks out that merge preview. The previous Action built `git diff base head` yet wrote `github.sha` as the claim subject and targeted GitHub checks at `github.sha`. These differ when the merge tree differs from the PR head tree. A reliable system must never silently treat them as one object. + +## Bounded new behavior + +`scripts/prepare_verified_change_input.py`, invoked from `action.yml` after checkout, now: + +1. Requires the actual checked-out commit to equal the full `github.sha` provided by the GitHub event. In the ordinary PR path, reads the commit's raw parent headers and requires precisely two parents: event `base.sha` and `pull_request.head.sha`, in that order. +2. Requires the base commit object to be available (attempting an explicit shallow fetch otherwise), with **no fallback** to `HEAD^ HEAD` or a different comparison on fetch failure. Computes `git diff --no-ext-diff --binary event-base checkout-merge` and its SHA-256. For non-PR push, compares the checked-out first parent to the checked-out commit. A root commit uses `git show`. +3. Writes a versioned `ovk-source-binding.json` sidecar recording the two event identities, evaluated commit/tree identity, exact diff SHA-256, input comparison mode, explicit limits on authentication/relying policy and whether input was structurally checked against the checkout. +4. Rejects mismatching or malformed commit IDs, incompatible PR parentage, and `pull_request_target`. Caller-selected change input remains supported **in advisory mode only**, and the sidecar marks it `caller_supplied_unbound`. A caller-selected input is refused in strict mode. To prevent an unbound advisory fixture from becoming a misleading shareable check or comment, the Action suppresses automatic GitHub Check and PR-comment emission for caller-selected input, and emits an explicit warning. The local advisory evidence can still be inspected as an experiment. +5. Makes this sidecar available among the existing Action artifacts. The **15 October real-PR dogfood exposed a second source-subject defect:** `build_repository_context` was overriding the Action's explicit `github.sha` argument with the contributor's `pull_request.head.sha`. The sidecar correctly identified the merge preview, while the evidence bundle incorrectly named the PR head. Earlier source-binding unit and Action dogfood tests did not compare the two outputs. This mismatch was reproduced from the uploaded Action artifact for draft PR #227 and is not retroactively repaired by PR #223/#224/#225/#226. +6. The subsequent **draft #227** adds the opt-in `ovk check --source-binding` path: validate event/base/head/merge/tree and the exact binary diff against the actual Git checkout before preserving the requested merge-preview evidence subject. Ordinary callers without the explicit binding retain the existing PR-head event interpretation. A real-Git adversarial regression and the read-only job-summary renderer check this correspondence. These are local observational checks, not an authenticated or signed subject certificate. + +The new test creates real base/head/merge Git commits, asserts the chosen binary diff, introduces wrong base/head/checkout identifiers, tests a contributor-only checkout, opaque inputs, `pull_request_target`, push, and failure without writing a misleading receipt. + +## Explicit manifest safety boundary + +The composite Action's `verification-manifest` mode accepts a caller-selected +manifest and checks the referenced materials under that manifest's own scope. +It does **not** execute the pull-request source-binding helper, so its +`subject.head_sha` alone does not establish that every changed PR construct +was assessed. Consequently an explicit manifest result is **local advisory +evidence only** in the Action: + +- The manifest path does not publish a GitHub Check or PR comment, regardless + of caller-supplied `emit-check` or `post-comment` inputs. +- `mode: strict` with `verification-manifest` is refused before evaluation. + Strict PR enforcement cannot be synthesized from an unbound manifest. +- The composite Action outputs are deliberately `decision_state=unknown`, + `recommendation=require_human_review`, and `exit_code=2` for any + manifest input, even if the selected manifest's own scoped verifier reports + `allow`. The same Action-level downgrade applies to any caller-supplied + `changed-files` diff. Only the internally prepared default change input + retains its scoped decision in Action outputs; neither form of unbound input + grants an `allow` output suitable for downstream PR approval. + The selected input's own result remains in local advisory artifacts. +- All reserved top-level Action outputs are invalidated before installation + (including if package installation fails and uploads run with `always()`). + Manifest mode deletes any prior source receipt/handoff and refuses a + nonempty existing output directory, preserving existing contents rather + than silently mixing cohorts. +- Local `ovk verify --manifest ... --advisory` remains an available + explicitly scoped experiment. Its results must identify which claims and + supplied materials were evaluated, and must not be presented as verification + of the entire PR or its source revision. + +This restriction is a publication-safety measure, not evidence authentication. +No existing signed source/evidence envelope, production approval, or +cross-repository certificate applicability follows. A future manifest +publication path must bind exact source revision, source-to-model coverage, +referenced material digests, results, and a qualified relying policy. + +## Important limitations + +- **Not source authenticity proof.** The GitHub event metadata, runner, checkout and storage remain unverified external observations. Raw parent shape is not a signed source/target validity certificate. +- **Not a signed evidence link.** This sidecar is **not yet incorporated into the canonical signed `EvidenceBundle` or `ovk-attestation-envelope`**. Its flags `bound_into_signed_evidence_envelope=false` and `production_reliance_authorized=false` are literal. Do not publish it as a portable signed verified-PR receipt. +- **Not general source correctness.** Diff inspection, compiled Assurance IR coverage and checker trust are separate. A structurally bound diff establishes *which Git tree was compared*, not that a machine claim comprehensively represents application source behavior. +- **Not PR-head verification.** The admitted default compares the event **base** to the synthetic **merge preview**. GitHub Checks created using `github.sha` are attributed to that merge SHA. For reviewer-facing PR-head checks, independently model *presentation SHA* and *verified subject SHA* before changing the Checks API target. +- **Shallow and private checkout prerequisites.** The default one-commit checkout may lack the base object. An explicit `fetch-depth: 2` on `actions/checkout` is recommended for a normal merge-preview PR; on some private/fork checkouts a missing base will require authenticated fetching. If absent and unfetchable, the Action fails closed. The first-user templates must be updated to guarantee this on the actual published Action revision. +- **Existing strict-profile no-go persists.** Historical issuer authorization, source-profile calibration, revocation, exact release identity and trustworthy evidence transport remain independently open under #25, #178, #195 and #205. No new `VALID`, production reuse, or automatic merge authorization. + +## Adversarial qualification + +Run `pytest -q tests/test_action_pr_source_binding_v1.py` and full repository CI on the exact PR SHA. The normal `action_default_diff_dogfood` workflow must pass from a genuine GitHub pull request event, exercising the new strict parent binding. A test pass on an earlier SHA does not authorize a later commit. + +The next qualified iteration will integrate this sidecar's exact comparison/identity into the signed evidence envelope with a typed provenance contract and distinguish source-subject identity from GitHub Check presentation identity, including base drift, PR updates, and fork permissions. That later work requires independent oracle evaluation and reviewer usefulness. This PR is a bounded correctness prerequisite for the same open-source CLI/Action product, **not** a new speculative subsystem. diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 2ccb800dc..5052abfb2 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -323,7 +323,7 @@ Consumers should trust only the Publish workflow path, exact immutable tag ident ## Historical release evidence -Historical releases such as `v1.2.1` remain immutable and retain their original evidence. Their workflow runs, release artifacts, and Sigstore identities are historical facts, not evidence for `v1.3.0-rc.1`. +Historical published releases such as `v1.2.1` retain their original release-artifact and workflow evidence, but **ref immutability is not established**. On 2026-10-15 GitHub reported the annotated `v1.2.1` tag as `unsigned`, the release's `immutable` field as `false`, and no active repository rulesets; the live `main` branch object reports `protected=false`. Verify each release asset, signed bundle, and immutable **commit SHA** separately. Historical workflows and Sigstore identities do not authorize `v1.3.0-rc.1`. Independent consumer repositories currently used by the consumer-pin workflow are: diff --git a/docs/RELEASE_NOTES_v1.2.1.md b/docs/RELEASE_NOTES_v1.2.1.md index ebded07a2..e6415be42 100644 --- a/docs/RELEASE_NOTES_v1.2.1.md +++ b/docs/RELEASE_NOTES_v1.2.1.md @@ -1,10 +1,10 @@ # OVK v1.2.1 -Patch release to close the immutable-tag keyless Sigstore production pin and fix release-gate regressions on `main`. +Historical patch release associated with keyless Sigstore artifact signing; Git-tag immutability was not established. Changes include release-gate regressions on `main`. ## Changes -- Publish workflow on an immutable `refs/tags/v1.2.1` Release: build → keyless cosign → verify → attach `*.cosign.bundle.json` +- Publish workflow associated with the published `v1.2.1` release: build → keyless cosign → verify → attach `*.cosign.bundle.json`. **Correction (2026-10-15):** the GitHub annotated tag object reports `verification.verified=false` (`unsigned`) and the release's `immutable` API field is `false`. Treat artifact Sigstore bundles separately from Git tag signing and ref protection. - Composite Action installs sync `ovk/package_data` before `pip install .` so policy schemas ship into site-packages - Align unit tests with honest CBMC template-harness limits, relative provenance URIs, and reliability-based repo-memory priors @@ -26,7 +26,7 @@ pip install -e '.[dev]' env: OVK_PACKAGE_VERSION: "1.2.1" steps: - - uses: fraware/open-verification-kernel@v1.2.1 + - uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 # exact historical source commit ``` ## Consumer Sigstore verify diff --git a/docs/SOURCE_SCOPED_JOB_SUMMARY.md b/docs/SOURCE_SCOPED_JOB_SUMMARY.md new file mode 100644 index 000000000..96c0954ff --- /dev/null +++ b/docs/SOURCE_SCOPED_JOB_SUMMARY.md @@ -0,0 +1,102 @@ +# Read-only source-scoped PR summary — developer preview + +**Status: unmerged development feature, not part of published `v1.2.1`.** +The historical Action does not contain this feature. This document records a +candidate [read-only summary workflow](https://github.com/fraware/open-verification-kernel/pull/227), +pending its exact-commit CI, source-path review, release governance and a +consenting independent adoption experiment. No production relying authority +or externally calibrated verification claim is established. + +## Reviewer experience + +A consenting repository enables the OVK composite Action on an ordinary +`pull_request` workflow with: + +- a full-SHA-reviewed Action revision and `actions/checkout` at the normal + GitHub merge-preview event SHA, with a sufficiently deep checkout; +- `mode: advisory`, `use-check: "true"`, **no** explicit `changed-files` + or `verification-manifest` input, `emit-check: "false"`, + `post-comment: "false"`; +- job `permissions: {contents: read}` and no untrusted PR secrets. + +Only in that bounded default path, the Action reads the local +`ovk-evidence.json` and `ovk-source-binding.json`. It rejects absent, +unbound, structurally inconsistent or mismatched source identifiers before +appending a concise summary to GitHub's native `GITHUB_STEP_SUMMARY` file. +This uses the existing workflow's job-summary presentation; no PR comment, +GitHub Check-write permission, third-party service, or hosted account is +needed. + +The summary includes the *recorded scoped* checker decision, the observed +merge-preview, original PR-head and base SHAs, SHA-256 of the observed diff +and of the exact local evidence bytes. It states that source/event +authenticity, complete source-to-model correspondence, signed envelope +binding, production reliance, and whole-PR security **are not established**. +A recorded `allow` never authorizes merging. No user-controlled source +snippet or raw code is copied into the summary. + +The job summary is a *review-navigation and evidence-identification artifact*, +not a signed certificate. Its identifiers are useful for comparing local +reports and locating the underlying artifacts. A SHA-256 digest is tamper +evidence only relative to a trusted expected digest; it does not establish +the authenticity of the evidence or the GitHub runner. + +**Portable human-review artifact (unreleased draft #228):** The same validated +text is also saved as `ovk-advisory-pr-summary.md` and included in the +existing OVK evidence-artifact upload. A teammate can inspect the report with +its exact source/decision limitations without copying the generic +`ovk-pr-comment.md`, which lacks these prominent source-binding caveats. +The standalone Markdown report is *not* a cryptographic certificate, and the +upload does not itself authenticate the file or event. Failed source validation +deletes any stale portable report before an always-running upload occurs. +The renderer deliberately excludes raw PR diffs and source excerpts from its +Markdown; the recorded digest supports subsequent independent comparison. +Native GitHub job-summary presentation remains available in parallel. + +## Reuse across repositories and CI systems + +The renderer itself uses only Python's standard library: + +```bash +python scripts/render_source_bound_job_summary.py \ + --evidence ovk-evidence.json \ + --source-binding ovk-source-binding.json \ + --diff .ovk-pr.diff \ + --output advisory-summary.md \ + --portable-output ovk-advisory-pr-summary.md +``` + +The evidence, source receipt, and **exact binary diff bytes** must come +from the same ordinary, internally prepared source-bound OVK PR run. The +renderer recomputes the input diff digest and refuses a changed or missing +diff; it also checks the complete source tree SHA field and agreement of +the evidence subject with the observed merge preview. Copying fixtures, passing caller-selected diffs, +relabeling PR heads, or using a multi-lane manifest is deliberately refused. +The renderer appends instead of overwriting an existing summary, so +multiple independent tools in the same CI job retain their own sections. +The file can be used in a different CI provider only if the exact local +observation contract is reproduced and checked; the script itself does not +authenticate third-party events. + +Use this first as a **read-only advisory preview**, strictly with maintainer +consent and human review. Existing public release controls in issue #25 and +source-model/refinement qualification in issue #195 remain separate. + +## Distribution and utility measurement + +Track the adoption chain prospectively, with source evidence and no inflated +marketing counts: + +1. consented outside-repository installation and setup time; +2. first natural PR with a complete, intelligible, source-identified + observation and explicit unsupported/unknown cases; +3. an independent reviewer reading the summary and the underlying evidence, + including reviewer minutes, useful findings and erroneous alerts; +4. a second natural PR and independent teammate adoption; +5. legitimate cross-repository reuse of an explicitly specified claim, with + applicability conditions and independent review, not a copy-pasted PASS. + +Treat the job summary's page views and GitHub stars as distribution indicators, +not as formal assurance, effective vulnerability detection, or economic value. +A pilot with no qualifying claims is a valid null result; report coverage and +attrition rather than claiming a perfect false-positive or false-negative rate. diff --git a/docs/benchmarks/adoption-summary.json b/docs/benchmarks/adoption-summary.json index c7ad230ba..861eab86a 100644 --- a/docs/benchmarks/adoption-summary.json +++ b/docs/benchmarks/adoption-summary.json @@ -1,47 +1,47 @@ { "schema_version": "ovk.adoption_summary.v1", "ovk_version": "1.2.1", - "updated_at": "2026-07-25T17:11:57Z", + "updated_at": "2026-10-15T23:46:39Z", "formal_pr_bench": { "cases_total": 132, "cases_passed": 132, - "pass_rate": 1.0, - "intent_recall": 1.0, + "pass_rate": 1, + "intent_recall": 1, "by_category": { "adversarial": { "cases_total": 2, "cases_passed": 2, - "pass_rate": 1.0 + "pass_rate": 1 }, "intent_recall": { "cases_total": 3, "cases_passed": 3, - "pass_rate": 1.0 + "pass_rate": 1 }, "lane": { "cases_total": 100, "cases_passed": 100, - "pass_rate": 1.0 + "pass_rate": 1 }, "multi_backend": { "cases_total": 1, "cases_passed": 1, - "pass_rate": 1.0 + "pass_rate": 1 }, "real_diff": { "cases_total": 18, "cases_passed": 18, - "pass_rate": 1.0 + "pass_rate": 1 }, "repair_loop": { "cases_total": 4, "cases_passed": 4, - "pass_rate": 1.0 + "pass_rate": 1 }, "routing": { "cases_total": 4, "cases_passed": 4, - "pass_rate": 1.0 + "pass_rate": 1 } }, "timing_ms": { @@ -50,14 +50,14 @@ "max": 34.02709299999884 } }, - "real_diff_recall": 1.0, + "real_diff_recall": 1, "pilot_dogfood": { "source": "local", "last_run": "2026-07-25T17:11:57Z", "manifests_passed": 8, "manifests_total": 8, "median_elapsed_ms": 4.71550000111165, - "false_positive_rate": 0.0, + "false_positive_rate": null, "external_manifest": "examples/pilot_repos/external_oss_ci_secrets.json", "weekly_schedule": "0 7 * * 1", "workflow": ".github/workflows/pilot-dogfood.yml", @@ -75,14 +75,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 95.62, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/fastapi-terraform/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "fraware/ovk-consumer-express-actions", @@ -95,14 +95,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 81.71, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/express-actions/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "in-repo/ovk-pilot-infra-terraform-k8s", @@ -115,14 +115,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 3, "prs_blocked": 2, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 46.14, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": "docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml", "evidence_url": "docs/pilots/infra-terraform-k8s/pilot-report.json", - "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md." + "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "TBD - recruiting first true external OSS adopter", diff --git a/docs/benchmarks/external-pilots-registry.json b/docs/benchmarks/external-pilots-registry.json index 01a6beb62..750f5d87b 100644 --- a/docs/benchmarks/external-pilots-registry.json +++ b/docs/benchmarks/external-pilots-registry.json @@ -1,6 +1,6 @@ { "schema_version": "ovk.external_pilots_registry.v1", - "updated_at": "2026-07-25T17:08:12Z", + "updated_at": "2026-10-15T23:46:39Z", "external_pilots": [ { "repository": "fraware/ovk-consumer-fastapi-terraform", @@ -13,14 +13,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 95.62, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/fastapi-terraform/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "fraware/ovk-consumer-express-actions", @@ -33,14 +33,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 2, "prs_blocked": 1, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 81.71, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-advisory-pr.yml", "evidence_url": "docs/pilots/express-actions/pilot-report.json", - "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md." + "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "in-repo/ovk-pilot-infra-terraform-k8s", @@ -53,14 +53,14 @@ "advisory_end": "2026-07-25", "prs_evaluated": 3, "prs_blocked": 2, - "false_positives": 0, - "false_positive_rate": 0.0, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": 46.14, "strict_enabled": false, "ovk_version_pin": "1.2.1", "workflow_path": "docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml", "evidence_url": "docs/pilots/infra-terraform-k8s/pilot-report.json", - "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md." + "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md. 2026-10-15 correction: historical fixture/dogfood zero-error labels have no independent reviewer ground truth; reported as unknown. Original values remain in git history." }, { "repository": "TBD - recruiting first true external OSS adopter", diff --git a/docs/pilots/express-actions/REPORT.md b/docs/pilots/express-actions/REPORT.md index 094a29d98..1a530a99d 100644 --- a/docs/pilots/express-actions/REPORT.md +++ b/docs/pilots/express-actions/REPORT.md @@ -8,12 +8,15 @@ Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +**Historical-data interpretation correction (2026-10-15):** The original companion JSON files preserve fixture counts and earlier self-reported error fields for historical inspection. Those fields have no independently adjudicated ground truth; the normalized external-pilot registry and adoption summary now record the rates as **null/unknown**. This clarification is not a new measurement. + ## Repository profile | Field | Value | |---|---| | Stack | TypeScript Express service with GitHub Actions workflows | -| Role | Independent maintained consumer gate (program section 23) | +| Role | Maintained consumer fixture validation (not independent adoption) | | Advisory workflow | `.github/workflows/ovk-advisory-pr.yml` | | Pilot ledger | Consumer `pilot/ledger.json` (automated_scenario rows only) | | Production gate | `production_gate_met: false` | @@ -40,14 +43,14 @@ Documented playbook path reproduced locally against the consumer clone: | Metric | Measured | |---|---| -| False positives | **0** | -| False positive rate | **0.0** (0/2 fixture cases) | +| Independently adjudicated false positives | **Unknown** — fixture decisions lack independent reviewer ground truth | +| Independently adjudicated false-positive rate | **Unknown** — 2 fixtures do not estimate an external error rate | ## False negatives | Metric | Measured | |---|---| -| False negatives | **0** | +| Independently adjudicated false negatives | **Unknown** — only specified known-bad fixtures were exercised | Unsafe secrets-on-PR fixture blocked as expected. @@ -70,13 +73,13 @@ Live production unknown rate: **unmeasured**. ## Configuration changes -- Pin Action to `fraware/open-verification-kernel@v1.2.1`. +- Historical workflow used `@v1.2.1`; for any newly consented advisory experiment, review and pin the full source commit `a27d5720f4350c00bca34f71d991c31f5a2f38c7` and its transitive dependencies. The historical tag is unsigned/unprotected. - Advisory mode + `default_on_unknown: require_human_review`. - `.verification/ci_secrets_pilot.json` in the consumer; in-repo mirror `examples/pilot_repos/express_actions_consumer.json`. - `post-comment` / `emit-check` exercised on the advisory PR workflow path. ## Strict-mode recommendation -**Remain advisory (`remain_advisory`).** Fixture metrics meet the playbook FP target numerically, but without a live human-adjudicated advisory window this report does not authorize strict branch protection. +**Remain advisory (`remain_advisory`).** These controlled fixture outcomes do not establish an externally adjudicated error rate. Without a live, independently reviewed advisory cohort and qualified source/profile and release controls, strict branch protection is unauthorized. Median `ovk check` latency on fixture diffs: ~82 ms. diff --git a/docs/pilots/fastapi-terraform/REPORT.md b/docs/pilots/fastapi-terraform/REPORT.md index 53c677628..76cb64080 100644 --- a/docs/pilots/fastapi-terraform/REPORT.md +++ b/docs/pilots/fastapi-terraform/REPORT.md @@ -8,12 +8,15 @@ Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +**Historical-data interpretation correction (2026-10-15):** The original companion JSON files preserve fixture counts and earlier self-reported error fields for historical inspection. Those fields have no independently adjudicated ground truth; the normalized external-pilot registry and adoption summary now record the rates as **null/unknown**. This clarification is not a new measurement. + ## Repository profile | Field | Value | |---|---| | Stack | FastAPI web app with Terraform infrastructure | -| Role | Independent maintained consumer gate (program section 23) | +| Role | Maintained consumer fixture validation (not independent adoption) | | Advisory workflow | `.github/workflows/ovk-advisory-pr.yml` | | Pilot ledger | Consumer `pilot/ledger.json` (automated_scenario rows only) | | Production gate | `production_gate_met: false` (no 30 human adjudications yet) | @@ -40,8 +43,8 @@ Documented playbook path reproduced locally against the consumer clone: | Metric | Measured | |---|---| -| False positives | **0** | -| False positive rate | **0.0** (0/2 fixture cases) | +| Independently adjudicated false positives | **Unknown** — fixture decisions lack independent reviewer ground truth | +| Independently adjudicated false-positive rate | **Unknown** — 2 fixtures do not estimate an external error rate | Passing workflow fixture was allowed; no incorrect blocks on the known-good case. @@ -49,7 +52,7 @@ Passing workflow fixture was allowed; no incorrect blocks on the known-good case | Metric | Measured | |---|---| -| False negatives | **0** | +| Independently adjudicated false negatives | **Unknown** — only specified known-bad fixtures were exercised | Known-bad secrets-on-`pull_request` fixture was blocked as expected (100% block rate on that unsafe fixture). @@ -73,13 +76,13 @@ No live PR adjudications yet — unknown appropriateness for production traffic ## Configuration changes -- Pin Action to `fraware/open-verification-kernel@v1.2.1` (immutable tag / audited SHA only). +- Historical source identity: `fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7`. The v1.2.1 tag was unsigned and unprotected in the 2026-10-15 audit; the original workflow used `@v1.2.1` and is an archival example, not an approved immutable installation. - Advisory mode via `.verification/config.yml` (`mode: advisory`, `default_on_unknown: require_human_review`). - Pilot manifest under `.verification/ci_secrets_pilot.json` (consumer) mirrored by in-repo `examples/pilot_repos/fastapi_terraform_consumer.json`. - Artifact upload of evidence / comment outputs for ingest. ## Strict-mode recommendation -**Remain advisory (`remain_advisory`).** Fixture FP rate is under 5%, but this publication is maintained-consumer dogfood without human-adjudicated live PRs. Do not enable strict required checks on protected branches from this evidence alone. +**Remain advisory (`remain_advisory`).** Known-good and known-bad fixture decisions matched their expected behavior, but no independently adjudicated live PR cohort or external false-positive rate exists. Strict required checks remain unauthorized. Median `ovk check` latency on fixture diffs: ~96 ms. diff --git a/docs/pilots/infra-terraform-k8s/REPORT.md b/docs/pilots/infra-terraform-k8s/REPORT.md index d1135f0f3..a535e858c 100644 --- a/docs/pilots/infra-terraform-k8s/REPORT.md +++ b/docs/pilots/infra-terraform-k8s/REPORT.md @@ -9,6 +9,9 @@ Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +**Historical-data interpretation correction (2026-10-15):** The original companion JSON files preserve fixture counts and earlier self-reported error fields for historical inspection. Those fields have no independently adjudicated ground truth; the normalized external-pilot registry and adoption summary now record the rates as **null/unknown**. This clarification is not a new measurement. + ## Repository profile | Field | Value | @@ -31,8 +34,8 @@ This pilot satisfies OVK-PR8’s third-pilot requirement at the **advisory metri | Metric | Measured | |---|---| -| False positives | **0** | -| False positive rate | **0.0** (0/3 fixture cases) | +| Independently adjudicated false positives | **Unknown** — fixture decisions lack independent reviewer ground truth | +| Independently adjudicated false-positive rate | **Unknown** — 3 fixtures do not estimate an external error rate | Private / repaired infrastructure fixtures allowed when expected. @@ -40,7 +43,7 @@ Private / repaired infrastructure fixtures allowed when expected. | Metric | Measured | |---|---| -| False negatives | **0** | +| Independently adjudicated false negatives | **Unknown** — only specified known-bad fixtures were exercised | Public-sensitive and failing repair-loop fixtures blocked as expected. @@ -66,6 +69,6 @@ Public-sensitive and failing repair-loop fixtures blocked as expected. ## Strict-mode recommendation -**Remain advisory (`remain_advisory`).** Publish-only scaffold with fixture metrics. Promote to strict only after a live infra consumer completes the playbook window with FP under 5% and human adjudication. +**Remain advisory (`remain_advisory`).** Publish-only scaffold with fixture metrics. Strict enforcement remains a separate qualification decision contingent on independently adjudicated natural PRs, source-profile coverage, unknown and miss rates, provenance, reviewer approval, and rollback controls. No fixture-based percentage authorizes promotion. Median `ovk check` latency on fixture diffs: ~46 ms. diff --git a/docs/templates/external_pilot_report.template.json b/docs/templates/external_pilot_report.template.json index 4ef983679..3e741171b 100644 --- a/docs/templates/external_pilot_report.template.json +++ b/docs/templates/external_pilot_report.template.json @@ -2,17 +2,19 @@ "schema_version": "ovk.external_pilot_report.v1", "repository": "org/example-repo", "repository_url": "https://github.com/org/example-repo", - "check_types": ["ci_secrets"], - "advisory_start": "2026-01-01", - "advisory_end": "2026-01-15", - "prs_evaluated": 0, - "prs_blocked": 0, - "false_positives": 0, - "false_positive_rate": 0.0, + "check_types": [ + "ci_secrets" + ], + "advisory_start": null, + "advisory_end": null, + "prs_evaluated": null, + "prs_blocked": null, + "false_positives": null, + "false_positive_rate": null, "median_check_latency_ms": null, "strict_enabled": false, - "ovk_version_pin": "1.2.0", + "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-pilot.yml", "evidence_url": null, - "notes": "Self-reported metrics from the external pilot maintainer. Ingest with scripts/ingest_external_pilot_metrics.py." + "notes": "Template only. Leave null until measured. False-positive counts/rates require separate reviewer adjudication with explicit denominator and evidence location; tool PASS/BLOCK is not ground truth." } diff --git a/examples/github_workflows/branch_protection_required_check.yml b/examples/github_workflows/branch_protection_required_check.yml index 76e2668c5..6edbca77d 100644 --- a/examples/github_workflows/branch_protection_required_check.yml +++ b/examples/github_workflows/branch_protection_required_check.yml @@ -1,6 +1,10 @@ name: OVK Branch Protection (Strict Required Check) -# Phase 2+: strict enforcement with required check "Open Verification Kernel". +# FUTURE REFERENCE ONLY — NOT INSTALLABLE UNTIL REPLACED AND AUTHORIZED. +# Strict enforcement is prohibited for current executable_advisory source profiles. +# Only qualify this after independent external calibration of the exact claim +# family, protected source acquisition, live policy, authorized package and +# a repository-owner decision. Do not copy this template into active CI. # Add the check name to branch protection after this workflow runs green on a sample PR. on: @@ -12,8 +16,8 @@ permissions: checks: write env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + # Placeholder, deliberately not a published/working version. + OVK_PACKAGE_VERSION: "REPLACE_WITH_AUTHORIZED_VERSION" jobs: ovk-required-check: @@ -27,7 +31,7 @@ jobs: git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: Run OVK strict check (required) - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@REPLACE_WITH_VERIFIED_SHA with: mode: strict use-check: "true" diff --git a/examples/github_workflows/external_consumer.yml b/examples/github_workflows/external_consumer.yml index 572bbebb8..c76f21362 100644 --- a/examples/github_workflows/external_consumer.yml +++ b/examples/github_workflows/external_consumer.yml @@ -1,7 +1,12 @@ -name: OVK External Consumer Example +name: OVK External Advisory Consumer -# Phase 2: strict enforcement with emit-check for production adoption. -# See also: pilot_fork_adopter.yml (phase 0), pilot_advisory_with_comment.yml (phase 1). +# Minimum-permission demonstration for a consenting engineering team. +# Copy to .github/workflows/ovk-advisory.yml and keep non-blocking. +# Historical v1.2.1 installs can fall back to the Action checkout if the +# requested package cannot be installed. The exact Action source SHA is +# pinned, but this old version does NOT supply a fail-closed package identity. +# The new contract in draft PR #220 is unpublished; see release status. +# Do not make this advisory check a required merge gate. on: pull_request: @@ -9,35 +14,23 @@ on: permissions: contents: read - pull-requests: write - checks: write env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + OVK_PACKAGE_VERSION: "1.2.1" jobs: - ovk-check: + ovk-advisory: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Run Open Verification Kernel check path - uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: - mode: strict - use-check: "true" - emit-check: "true" - changed-files: examples/ci_secrets/workflow_secrets_on_pr.diff - post-comment: "false" - - ovk-full-mvp: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run Open Verification Kernel (all five lanes) - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + persist-credentials: false + - name: Run bounded OVK PR checks + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory - verification-manifest: examples/verification_manifests/full_mvp.json - bundle-output-dir: ovk-full-mvp-bundle + use-check: "true" + emit-check: "false" post-comment: "false" + # The Action uploads structured evidence artifacts. A green job in + # advisory mode is not proof that the underlying result is an allow. diff --git a/examples/github_workflows/pilot_advisory_with_comment.yml b/examples/github_workflows/pilot_advisory_with_comment.yml index 6523b0324..cf8298b05 100644 --- a/examples/github_workflows/pilot_advisory_with_comment.yml +++ b/examples/github_workflows/pilot_advisory_with_comment.yml @@ -1,6 +1,8 @@ name: OVK Pilot (Advisory + Check) # Phase 1: advisory mode with GitHub check observability (no merge blocking). +# For fork PRs GitHub normally issues read-only tokens; write operations are +# best-effort. Set post-comment: false for untrusted forks. # Copy to .github/workflows/ovk-advisory-check.yml in your repository. on: @@ -13,8 +15,10 @@ permissions: checks: write env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + # Historical v1.2.1 distribution. This old Action has a fallback to checkout + # if package installation fails; pinning does not establish package-byte identity. + # For consenting advisory evaluations only; see release status before use. + OVK_PACKAGE_VERSION: "1.2.1" jobs: ovk-advisory-check: @@ -22,7 +26,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: OVK advisory check with emitted GitHub check - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory use-check: "true" diff --git a/examples/github_workflows/pilot_fork_adopter.yml b/examples/github_workflows/pilot_fork_adopter.yml index f1dea0edf..a690c897c 100644 --- a/examples/github_workflows/pilot_fork_adopter.yml +++ b/examples/github_workflows/pilot_fork_adopter.yml @@ -2,7 +2,8 @@ name: OVK Pilot (Fork Adopter) # Copy to .github/workflows/ovk-pilot.yml in your repository. # Phase 0: advisory check + manifest verify; upload metrics for maintainer ingest. -# See docs/EXTERNAL_PILOT_PLAYBOOK.md for advisory → strict rollout. +# Advisory-only pilot. Strict enforcement is a separate qualification gate. +# See docs/EXTERNAL_PILOT_PLAYBOOK.md for supported claims and reviewer protocol. on: pull_request: @@ -10,44 +11,48 @@ on: permissions: contents: read - pull-requests: write + pull-requests: read env: - # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). - OVK_PACKAGE_VERSION: "1.3.0-rc.1" + # Historical v1.2.1 Action has fallback to checkout if package installation fails. + # The full source SHA below pins only the top-level Action, not its transitive dependencies. + # Consent-based, advisory use only: its manually supplied diff is not source-bound. + OVK_PACKAGE_VERSION: "1.2.1" jobs: ovk-ci-secrets-pilot: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 + persist-credentials: false - name: Build PR diff for OVK run: | git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: OVK advisory check - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory use-check: "true" changed-files: ovk-pr.diff + emit-check: "false" post-comment: "false" - name: OVK advisory verify (ci_secrets lane manifest) - uses: fraware/open-verification-kernel@v1.3.0-rc.1 + uses: fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7 with: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json bundle-output-dir: ovk-pilot-bundle - post-comment: "true" + emit-check: "false" + post-comment: "false" - name: Upload pilot metrics artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: ovk-pilot-artifacts path: | ovk-evidence.json ovk-pilot-bundle/** - external_pilot_report.json retention-days: 30 - if-no-files-found: ignore + if-no-files-found: warn diff --git a/ovk/cli.py b/ovk/cli.py index bcf3970f4..398ff2a3d 100644 --- a/ovk/cli.py +++ b/ovk/cli.py @@ -726,11 +726,37 @@ def check( strict: bool = typer.Option(False, help="Enforce non-zero exit codes (overrides advisory)."), no_cache: bool = typer.Option(False, help="Disable lane result cache."), format: str = typer.Option("md", help="Output format: md, json, or github."), + source_binding: Optional[Path] = typer.Option( + None, "--source-binding", + help="Local PR merge-preview source receipt; checked against Git/event/diff before choosing evidence subject.", + ), ) -> None: """Infer, compile, and verify affected lanes for a PR diff.""" enforce_exit = strict or not advisory input_path = diff or changed_files files, diff_text = load_diff_or_changed_files(input_path) + if source_binding is not None: + if input_path is None or github_event is None: + raise typer.BadParameter("source binding requires an input file and GitHub PR event") + # A valid Git merge preview can have an empty diff. The byte-level + # receipt check still applies; lack of changed lines must not be + # mislabeled as a validated source-to-model PASS. + from ovk.core.source_binding_identity import ( + SourceSubjectMismatch, + require_bound_merge_preview_subject, + ) + + try: + require_bound_merge_preview_subject( + receipt_path=source_binding, + diff_path=input_path, + event_path=github_event, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + except (SourceSubjectMismatch, OSError, ValueError) as exc: + raise typer.BadParameter(f"refusing unbound evidence subject: {exc}") from exc meta = read_json_file(metadata) if metadata else None result = run_check( changed_files=files, @@ -741,6 +767,7 @@ def check( repo=repo, head_sha=head_sha, base_sha=base_sha, + preserve_explicit_subject=source_binding is not None, use_cache=not no_cache, ) output_dir.mkdir(parents=True, exist_ok=True) diff --git a/ovk/core/check.py b/ovk/core/check.py index 0d7c5b952..5e66a5732 100644 --- a/ovk/core/check.py +++ b/ovk/core/check.py @@ -51,6 +51,7 @@ def run_check( repo: str = "unknown/repo", head_sha: str = "unknown", base_sha: str | None = None, + preserve_explicit_subject: bool = False, cache_dir: Path | None = None, use_cache: bool = True, parallel: bool = True, @@ -67,6 +68,7 @@ def run_check( repo=repo, head_sha=head_sha, base_sha=base_sha, + preserve_explicit_subject=preserve_explicit_subject, cache_dir=cache_dir if cache_dir is not None else DEFAULT_CACHE_DIR, use_cache=use_cache, parallel=parallel, diff --git a/ovk/core/context.py b/ovk/core/context.py index c9cfb222e..87069f9e1 100644 --- a/ovk/core/context.py +++ b/ovk/core/context.py @@ -204,6 +204,7 @@ def build_repository_context( repo: str = "unknown/repo", head_sha: str = "unknown", base_sha: str | None = None, + preserve_explicit_subject: bool = False, ) -> RepositoryContext: """Build repository context from PR metadata and trusted policy materials.""" files = changed_files or [] @@ -214,7 +215,16 @@ def build_repository_context( defaults = metadata_to_self_protection_defaults(github) actor_type = str(defaults.get("actor_type", "unknown")) repo = github.repository or repo - head_sha = github.head_sha or head_sha + # A separately checked source-binding receipt permits an observed + # merge-preview to be the subject of a default PR diff. Ordinary + # event-based callers retain the historical PR-head convention. + if preserve_explicit_subject: + if not head_sha or head_sha == "unknown": + raise ValueError("bound source subject requires explicit checkout SHA") + branch_metadata["event_pr_head_sha"] = github.head_sha + branch_metadata["source_subject_origin"] = "locally_bound_merge_preview" + else: + head_sha = github.head_sha or head_sha base_sha = github.base_sha or base_sha branch_metadata["github"] = defaults if check_metadata_path is not None and check_metadata_path.exists(): diff --git a/ovk/core/kernel.py b/ovk/core/kernel.py index 85965b572..a43bdde89 100644 --- a/ovk/core/kernel.py +++ b/ovk/core/kernel.py @@ -90,6 +90,7 @@ def execute_kernel( repo: str = "unknown/repo", head_sha: str = "unknown", base_sha: str | None = None, + preserve_explicit_subject: bool = False, budget: VerificationBudget | None = None, cache_dir: Path | None = DEFAULT_CACHE_DIR, use_cache: bool = True, @@ -113,6 +114,7 @@ def execute_kernel( repo=repo, head_sha=head_sha, base_sha=base_sha, + preserve_explicit_subject=preserve_explicit_subject, ) compiler = ObligationCompilerRegistry.default() diff --git a/ovk/core/source_binding_identity.py b/ovk/core/source_binding_identity.py new file mode 100644 index 000000000..c7778bd1f --- /dev/null +++ b/ovk/core/source_binding_identity.py @@ -0,0 +1,108 @@ +"""Admit a *local observed* merge-preview subject for the OVK check CLI. + +The Action's source-input preparer has already checked the current Git checkout +and generated a receipt. This module independently compares that receipt to +the current checkout, the current PR event, and exact diff bytes before the +event-derived PR-head value may be replaced as the evidence subject. + +These are local structural checks, not authenticated GitHub provenance or +source-to-IR correctness. Never treat their success as a signed certificate. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import subprocess +from pathlib import Path +from typing import Any + + +SHA = re.compile(r"^[0-9a-f]{40}$") +DIGEST = re.compile(r"^[0-9a-f]{64}$") +SCHEMA = "ovk.pr_source_input_binding.v1-development" +COMPARISON = "event_base_to_verified_merge_preview" + + +class SourceSubjectMismatch(ValueError): + pass + + +def _read_object(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise SourceSubjectMismatch(f"expected object in {path.name}") + return data + + +def _sha(value: Any, name: str) -> str: + if not isinstance(value, str) or SHA.fullmatch(value) is None: + raise SourceSubjectMismatch(f"missing or malformed {name}") + return value + + +def _git_object(revspec: str) -> str: + try: + result = subprocess.run( + ["git", "rev-parse", "--verify", revspec], + check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, timeout=10, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise SourceSubjectMismatch("unable to inspect checked-out Git source") from exc + if result.returncode != 0: + raise SourceSubjectMismatch("unable to inspect checked-out Git source") + return _sha(result.stdout.strip(), "checked-out Git object") + + +def require_bound_merge_preview_subject( + *, receipt_path: Path, diff_path: Path, event_path: Path, + repo: str, head_sha: str, base_sha: str | None, +) -> None: + """Fail closed unless all locally observed source identities agree.""" + receipt = _read_object(receipt_path) + event = _read_object(event_path) + if receipt.get("schema_version") != SCHEMA or receipt.get("event_name") != "pull_request": + raise SourceSubjectMismatch("unsupported PR source observation contract") + if receipt.get("input_comparison") != COMPARISON: + raise SourceSubjectMismatch("input comparison is not bound to PR merge preview") + if receipt.get("input_structurally_bound_to_checkout") is not True: + raise SourceSubjectMismatch("change input was not structurally bound to checkout") + for key in ("independent_event_authentication", "bound_into_signed_evidence_envelope", + "production_reliance_authorized"): + if receipt.get(key) is not False: + raise SourceSubjectMismatch(f"unexpected reliance field: {key}") + + checkout = _sha(receipt.get("evaluated_checkout_commit_sha"), "checkout commit") + tree = _sha(receipt.get("evaluated_checkout_tree_sha"), "checkout tree") + subject = _sha(receipt.get("evidence_subject_sha"), "receipt evidence subject") + event_base = _sha(receipt.get("event_base_sha"), "event base") + event_head = _sha(receipt.get("event_pr_head_sha"), "event PR head") + requested = _sha(head_sha, "explicit requested evidence subject") + if subject != checkout or requested != checkout: + raise SourceSubjectMismatch("explicit evidence subject is not checked-out merge preview") + if _git_object("HEAD") != checkout or _git_object("HEAD^{tree}") != tree: + raise SourceSubjectMismatch("checked-out Git object differs from source receipt") + + pr = event.get("pull_request") + ev_repo = event.get("repository") + if not isinstance(pr, dict) or not isinstance(ev_repo, dict): + raise SourceSubjectMismatch("GitHub pull_request event identity is missing") + ev_base, ev_head = pr.get("base"), pr.get("head") + if not isinstance(ev_base, dict) or not isinstance(ev_head, dict): + raise SourceSubjectMismatch("GitHub event PR parent objects are missing") + if _sha(ev_base.get("sha"), "GitHub event base") != event_base: + raise SourceSubjectMismatch("GitHub event base differs from source receipt") + if _sha(ev_head.get("sha"), "GitHub event head") != event_head: + raise SourceSubjectMismatch("GitHub event PR head differs from source receipt") + if ev_repo.get("full_name") != repo: + raise SourceSubjectMismatch("GitHub event repository differs from requested repository") + if base_sha is not None and base_sha != event_base: + raise SourceSubjectMismatch("caller base differs from checked PR base") + + digest = receipt.get("input_diff_sha256") + if not isinstance(digest, str) or DIGEST.fullmatch(digest) is None: + raise SourceSubjectMismatch("missing or malformed exact diff digest") + if hashlib.sha256(diff_path.read_bytes()).hexdigest() != digest: + raise SourceSubjectMismatch("actual input diff bytes differ from source receipt") diff --git a/scripts/collect_pilot_metrics.py b/scripts/collect_pilot_metrics.py index ce9674f08..cb1cc4e4f 100644 --- a/scripts/collect_pilot_metrics.py +++ b/scripts/collect_pilot_metrics.py @@ -54,16 +54,6 @@ def _bundle_artifacts(bundle_dir: Path | None) -> list[str]: return sorted(path.name for path in bundle_dir.rglob("*") if path.is_file()) -def _false_positive_rate(results: list[dict[str, Any]]) -> float | None: - evaluated = [item for item in results if "passed" in item] - if not evaluated: - return None - false_positives = sum( - 1 for item in evaluated if not item.get("passed") and item.get("merge_recommendation") == "block" - ) - return false_positives / len(evaluated) - - def _median_elapsed_ms(results: list[dict[str, Any]]) -> float | None: timings = [float(item["elapsed_ms"]) for item in results if "elapsed_ms" in item] if not timings: @@ -190,7 +180,7 @@ def collect_pilot_metrics( "manifests_passed": manifests_passed, "manifests_total": manifests_total, "median_elapsed_ms": median_elapsed, - "false_positive_rate": _false_positive_rate(results), + "false_positive_rate": None, # no independent ground-truth adjudication in fixture output "block_rate_on_unsafe_fixture": block_rate, "external_manifest": _repo_relative(manifest_path), }, diff --git a/scripts/ingest_external_pilot_metrics.py b/scripts/ingest_external_pilot_metrics.py index bfb90662a..f1cd1d9dc 100644 --- a/scripts/ingest_external_pilot_metrics.py +++ b/scripts/ingest_external_pilot_metrics.py @@ -40,14 +40,6 @@ def _iso_timestamp() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") -def _find_file(artifacts_dir: Path, name: str) -> Path | None: - direct = artifacts_dir / name - if direct.is_file(): - return direct - matches = sorted(artifacts_dir.rglob(name)) - return matches[0] if matches else None - - def _median_elapsed_ms(results: list[dict[str, Any]]) -> float | None: timings = [float(item["elapsed_ms"]) for item in results if "elapsed_ms" in item] if not timings: @@ -55,16 +47,6 @@ def _median_elapsed_ms(results: list[dict[str, Any]]) -> float | None: return float(statistics.median(timings)) -def _false_positive_rate(results: list[dict[str, Any]]) -> float | None: - evaluated = [item for item in results if "passed" in item] - if not evaluated: - return None - false_positives = sum( - 1 for item in evaluated if not item.get("passed") and item.get("merge_recommendation") == "block" - ) - return false_positives / len(evaluated) - - def _blocked_count(results: list[dict[str, Any]]) -> int | None: evaluated = [item for item in results if "merge_recommendation" in item] if not evaluated: @@ -95,9 +77,10 @@ def _metrics_from_artifacts(artifacts_dir: Path) -> dict[str, Any]: results = list(pilot_report.get("results", [])) metrics["prs_evaluated"] = len(results) or None metrics["prs_blocked"] = _blocked_count(results) - metrics["false_positive_rate"] = _false_positive_rate(results) - if metrics["false_positive_rate"] is not None and metrics["prs_evaluated"]: - metrics["false_positives"] = round(metrics["false_positive_rate"] * metrics["prs_evaluated"]) + # 'passed' and 'merge_recommendation' are OVK's own output labels. + # They cannot independently adjudicate a false positive or measure + # false-negative risk. Only a separately reviewed external report may + # supply false_positives / false_positive_rate. metrics["median_check_latency_ms"] = _median_elapsed_ms(results) if evidence_path and evidence_path.exists(): @@ -111,24 +94,42 @@ def _metrics_from_artifacts(artifacts_dir: Path) -> dict[str, Any]: elif isinstance(timing, (int, float)): metrics["median_check_latency_ms"] = float(timing) - external_report = _find_file(artifacts_dir, "external_pilot_report.json") - if external_report is not None: - report = read_json_file(external_report) - for report_key, row_key in REPORT_FIELD_MAP.items(): - if report_key in report and report[report_key] is not None: - metrics[row_key] = report[report_key] + # Artifact directories are not an adjudication authority. Even if they + # contain an external_pilot_report.json, never auto-adopt its verdicts, + # repository identity or strict_enabled flag. A report is accepted only + # via the explicit, separately reviewed --report input. return metrics def _apply_report(row: dict[str, Any], report_path: Path) -> dict[str, Any]: + """Apply exactly one local report snapshot without reusing stale cohort data.""" report = read_json_file(report_path) + # The CLI --repo argument is authoritative. Check identity before copying + # any report values; no content here independently authenticates a reviewer. + if report.get("repository") and report["repository"] != row["repository"]: + raise ValueError("external pilot report repository mismatch") updated = dict(row) + if any(report.get(key) is not None for key in ("prs_evaluated", "prs_blocked")): + # Counts identify a potentially different cohort, regardless of + # whether the report includes fresh error/latency observations. + for field in ( + "prs_evaluated", "prs_blocked", "false_positives", + "false_positive_rate", "median_check_latency_ms", + ): + updated[field] = None + else: + # Even without new counts, do not join an independently supplied + # partial error observation to a stale counterpart. + for field, counterpart in ( + ("false_positives", "false_positive_rate"), + ("false_positive_rate", "false_positives"), + ): + if report.get(field) is not None and report.get(counterpart) is None: + updated[counterpart] = None for report_key, row_key in REPORT_FIELD_MAP.items(): if report_key in report and report[report_key] is not None: updated[row_key] = report[report_key] - if "repository" in report and report["repository"]: - updated["repository"] = report["repository"] return updated @@ -159,10 +160,29 @@ def build_pilot_row( if existing is not None: row.update({key: value for key, value in existing.items() if value is not None or key in row}) row["repository"] = repo - if report_path is not None and report_path.exists(): + if artifacts_dir is not None: + if not artifacts_dir.is_dir(): + raise FileNotFoundError(f"pilot artifacts directory not found: {artifacts_dir}") + # An explicitly ingested artifact directory is a new, potentially + # unreviewed cohort. Invalidate all prior cohort-specific metrics, + # including totals and timing, before reading whatever is available. + # An empty or partial artifact set must never inherit old counts. + for field in ( + "prs_evaluated", + "prs_blocked", + "false_positives", + "false_positive_rate", + "median_check_latency_ms", + ): + row[field] = None + artifact_metrics = _metrics_from_artifacts(artifacts_dir) + row.update({key: value for key, value in artifact_metrics.items() if value is not None}) + if report_path is not None: + if not report_path.is_file(): + raise FileNotFoundError(f"external pilot report not found: {report_path}") + # A selected report is still self-reported. Apply a single parsed + # snapshot with explicit cohort invalidation and identity checks. row = _apply_report(row, report_path) - if artifacts_dir is not None and artifacts_dir.is_dir(): - row.update({key: value for key, value in _metrics_from_artifacts(artifacts_dir).items() if value is not None}) row["status"] = _infer_status(row) return row diff --git a/scripts/prepare_verified_change_input.py b/scripts/prepare_verified_change_input.py new file mode 100644 index 000000000..4cb133acb --- /dev/null +++ b/scripts/prepare_verified_change_input.py @@ -0,0 +1,183 @@ +"""Construct source-bound Action diff with a separate non-authoritative receipt. + +For pull_request events the default checkout is a synthetic merge preview. +An observed base-to-merge diff must match the actual checked-out source. +Caller-provided inputs are explicitly unbound. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path + +SHA = re.compile(r"^[0-9a-f]{40}(?:[0-9a-f]{24})?$") + + +class BindingError(ValueError): + pass + + +def git(*args: str) -> bytes: + result = subprocess.run( + ["git", *args], capture_output=True, check=False, + ) + if result.returncode != 0: + raise BindingError( + f"git {args[0]} failed ({result.returncode}): " + f"{result.stderr.decode('utf-8', errors='replace')[:300]}" + ) + return result.stdout + + +def sha(value: str, required: bool = False) -> str | None: + if not value and not required: + return None + if not SHA.fullmatch(value): + raise BindingError("missing or malformed full commit SHA") + return value + + +def parents() -> list[str]: + # Raw commit headers retain parents even on shallow GitHub checkouts. + header = git("cat-file", "-p", "HEAD").split(b"\n\n", 1)[0] + result = [ + line.split(b" ", 1)[1].decode("ascii") + for line in header.splitlines() if line.startswith(b"parent ") + ] + if not all(SHA.fullmatch(item) for item in result): + raise BindingError("malformed parent in checked-out commit") + return result + + +def base_available(base: str) -> None: + test = subprocess.run( + ["git", "cat-file", "-e", f"{base}^{{commit}}"], + capture_output=True, check=False, + ) + if test.returncode != 0: + git("fetch", "--no-tags", "--depth=1", "origin", base) + git("cat-file", "-e", f"{base}^{{commit}}") + + +def prepare( + *, event_name: str, github_sha: str, base_sha: str = "", + pr_head_sha: str = "", changed_files: str = "", mode: str = "advisory", +) -> tuple[str, bytes, dict]: + checkout = git("rev-parse", "HEAD").decode().strip() + tree = git("rev-parse", "HEAD^{tree}").decode().strip() + if not SHA.fullmatch(checkout) or not SHA.fullmatch(tree): + raise BindingError("invalid checked-out Git object") + if checkout != sha(github_sha, required=True): + raise BindingError("checked-out commit differs from GITHUB_SHA") + if event_name == "pull_request_target": + raise BindingError("privileged pull_request_target is not admitted") + base = None + pr_head = None + if event_name == "pull_request": + base = sha(base_sha, required=True) + pr_head = sha(pr_head_sha, required=True) + if parents() != [base, pr_head]: + raise BindingError("PR merge parents disagree with event base and head") + base_available(base) + elif base_sha or pr_head_sha: + raise BindingError("PR identity parameters supplied for non-PR event") + + if changed_files: + if mode == "strict": + raise BindingError("caller-supplied diff cannot authorize strict review") + path = Path(changed_files) + if "\n" in changed_files or "\r" in changed_files or not path.is_file(): + raise BindingError("invalid or missing changed-files input") + diff = path.read_bytes() + input_path = str(path) + comparison = "caller_supplied_unbound" + bound = False + elif event_name == "pull_request": + diff = git("diff", "--no-ext-diff", "--binary", base, checkout) + input_path = ".ovk-pr.diff" + comparison = "event_base_to_verified_merge_preview" + bound = True + elif parents(): + diff = git("diff", "--no-ext-diff", "--binary", "HEAD^", "HEAD") + input_path = ".ovk-pr.diff" + comparison = "checked_out_first_parent_to_head" + bound = True + else: + diff = git("show", "--format=", "--no-ext-diff", "--binary", "HEAD") + input_path = ".ovk-pr.diff" + comparison = "checked_out_root_commit" + bound = True + + receipt = { + "schema_version": "ovk.pr_source_input_binding.v1-development", + "event_name": event_name, + "event_base_sha": base, + "event_pr_head_sha": pr_head, + "evaluated_checkout_commit_sha": checkout, + "evaluated_checkout_tree_sha": tree, + "evidence_subject_sha": checkout, + "input_comparison": comparison, + "input_diff_sha256": hashlib.sha256(diff).hexdigest(), + "input_structurally_bound_to_checkout": bound, + "independent_event_authentication": False, + "bound_into_signed_evidence_envelope": False, + "production_reliance_authorized": False, + } + return input_path, diff, receipt + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--event-name", required=True) + parser.add_argument("--github-sha", required=True) + parser.add_argument("--base-sha", default="") + parser.add_argument("--pr-head-sha", default="") + parser.add_argument("--changed-files", default="") + parser.add_argument("--mode", default="advisory", choices=("advisory", "strict")) + parser.add_argument("--input-path-output", type=Path, default=Path(".ovk-effective-input-path")) + parser.add_argument("--receipt-output", type=Path, default=Path("ovk-source-binding.json")) + args = parser.parse_args(argv) + try: + # These files are generated observations, never reusable input evidence. + # Clear earlier invocations before validating the current source identity. + output_paths = (args.input_path_output, args.receipt_output) + resolved_outputs = tuple(output.resolve() for output in output_paths) + output_alias = resolved_outputs[0] == resolved_outputs[1] + input_alias = bool(args.changed_files) and Path(args.changed_files).resolve() in resolved_outputs + for output in output_paths: + if output.is_dir() and not output.is_symlink(): + raise BindingError("generated output path is a directory") + output.unlink(missing_ok=True) + # Reject aliases only after removing the potentially stale reserved output. + if output_alias: + raise BindingError("handoff and receipt output paths must differ") + if input_alias: + raise BindingError("changed-files input overlaps generated output") + path, content, receipt = prepare( + event_name=args.event_name, github_sha=args.github_sha, + base_sha=args.base_sha, pr_head_sha=args.pr_head_sha, + changed_files=args.changed_files, mode=args.mode, + ) + except (BindingError, OSError, UnicodeError) as exc: + print(f"OVK source binding rejected: {exc}", file=sys.stderr) + return 2 + if path == ".ovk-pr.diff": + Path(path).write_bytes(content) + # The composite Action reads this file from Bash; a Windows CRLF suffix + # would become part of the path after command substitution. Emit exact LF. + args.input_path_output.write_bytes((path + "\n").encode("utf-8")) + args.receipt_output.write_text( + json.dumps(receipt, sort_keys=True, indent=2) + "\n", encoding="utf-8", + ) + print(f"OVK source input: {receipt['input_comparison']} [{receipt['input_diff_sha256']}]") + if not receipt["input_structurally_bound_to_checkout"]: + print("OVK warning: caller-supplied input is not bound to checked-out source") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_pilot_metrics.py b/scripts/render_pilot_metrics.py index d99b4f9a1..31c5b78be 100644 --- a/scripts/render_pilot_metrics.py +++ b/scripts/render_pilot_metrics.py @@ -122,7 +122,9 @@ def render_adoption_summary( "manifests_passed": pilot_dogfood.get("manifests_passed"), "manifests_total": pilot_dogfood.get("manifests_total"), "median_elapsed_ms": pilot_dogfood.get("median_elapsed_ms"), - "false_positive_rate": pilot_dogfood.get("false_positive_rate"), + # Legacy fixture PASS/BLOCK is not independent ground truth. + # No adjudication provenance is accepted on this dogfood path. + "false_positive_rate": None, "external_manifest": pilot_dogfood.get( "external_manifest", "examples/pilot_repos/external_oss_ci_secrets.json" ), diff --git a/scripts/render_source_bound_job_summary.py b/scripts/render_source_bound_job_summary.py new file mode 100644 index 000000000..748bd11e6 --- /dev/null +++ b/scripts/render_source_bound_job_summary.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Render a read-only GitHub job summary from source-scoped OVK PR observations. + +This renderer validates *structural agreement of local observations* only. +It cannot authenticate GitHub events, sign evidence, prove source-to-model +correspondence, or authorize relying on the observed checker decision. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + + +SHA40 = re.compile(r"^[0-9a-f]{40}$") +DIGEST64 = re.compile(r"^[0-9a-f]{64}$") +DECISIONS = frozenset({"allow", "block", "needs_review", "unknown", "error", "skipped"}) +COMPARISON = "event_base_to_verified_merge_preview" +SCHEMA = "ovk.pr_source_input_binding.v1-development" + + +class ObservationError(ValueError): + """An observed input identity cannot justify this informational summary.""" + + +def _json_object(path: Path) -> tuple[dict[str, Any], bytes]: + raw = path.read_bytes() + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ObservationError(f"expected JSON object: {path.name}") + return payload, raw + + +def build_summary( + evidence: dict[str, Any], + receipt: dict[str, Any], + evidence_bytes: bytes, + diff_bytes: bytes, +) -> str: + """Build a non-authoritative advisory report, rejecting unbound or stale input.""" + if receipt.get("schema_version") != SCHEMA: + raise ObservationError("unknown source observation contract") + if receipt.get("event_name") != "pull_request": + raise ObservationError("this summary is for GitHub PR merge previews only") + if receipt.get("input_comparison") != COMPARISON: + raise ObservationError("unrecognized or caller-selected input comparison") + if receipt.get("input_structurally_bound_to_checkout") is not True: + raise ObservationError("input was not structurally bound to the checkout") + if receipt.get("independent_event_authentication") is not False: + raise ObservationError("unexpected event-authentication status") + if receipt.get("bound_into_signed_evidence_envelope") is not False: + raise ObservationError("unexpected signed-envelope status") + if receipt.get("production_reliance_authorized") is not False: + raise ObservationError("unexpected reliance status") + + checkout = receipt.get("evaluated_checkout_commit_sha") + tree = receipt.get("evaluated_checkout_tree_sha") + subject = receipt.get("evidence_subject_sha") + head = receipt.get("event_pr_head_sha") + base = receipt.get("event_base_sha") + diff_hash = receipt.get("input_diff_sha256") + fields = {"checkout": checkout, "tree": tree, "subject": subject, "head": head, "base": base} + if any(not isinstance(v, str) or SHA40.fullmatch(v) is None for v in fields.values()): + raise ObservationError("missing or malformed source revision") + if not isinstance(diff_hash, str) or DIGEST64.fullmatch(diff_hash) is None: + raise ObservationError("missing or malformed diff digest") + if hashlib.sha256(diff_bytes).hexdigest() != diff_hash: + raise ObservationError("observed diff bytes disagree with receipt digest") + if checkout != subject: + raise ObservationError("receipt subject does not match evaluated checkout") + evidence_subject = evidence.get("subject") + if not isinstance(evidence_subject, dict) or evidence_subject.get("head_sha") != subject: + raise ObservationError("evidence subject disagrees with the observed checkout") + + decision = evidence.get("decision") + if not isinstance(decision, dict): + raise ObservationError("missing evidence decision") + state = decision.get("decision_state") + if state is None: + aliases = { + "allow": "allow", + "block": "block", + "require_human_review": "needs_review", + "needs_review": "needs_review", + "unknown": "unknown", + "error": "error", + "skipped": "skipped", + } + state = aliases.get(decision.get("merge_recommendation"), "unknown") + if state not in DECISIONS: + raise ObservationError("unknown evidence decision state") + + evidence_hash = hashlib.sha256(evidence_bytes).hexdigest() + return ( + "## Open Verification Kernel — advisory PR observation\n\n" + f"**Recorded scoped decision:** `{state}` (advisory; human review controls the PR).\n\n" + "| Observation | Exact value |\n" + "| --- | --- |\n" + f"| Evaluated GitHub merge-preview commit | `{checkout}` |\n" + f"| Observed Git tree identity | `{tree}` |\n" + f"| Original contributor PR head | `{head}` |\n" + f"| Event base revision | `{base}` |\n" + f"| Observed binary diff SHA-256 | `{diff_hash}` |\n" + f"| Local evidence file SHA-256 | `{evidence_hash}` |\n\n" + "**Interpretation:** This is an observation about a configured, bounded checker on " + "a GitHub merge-preview diff. The source relation was checked structurally in " + "the runner; neither the event nor the evidence is independently authenticated. " + "The diff receipt is not signed into the evidence envelope, and the result " + "does not establish whole-PR correctness, supported-source completeness, " + "or production relying authority. An `allow` is not permission to merge.\n\n" + "**Review next:** inspect `ovk-evidence.json`, `ovk-evidence-quality.json`, " + "and `ovk-source-binding.json` in this job's uploaded OVK artifact. " + "Carry exact source identities and assumptions into any further review.\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", type=Path, default=Path("ovk-evidence.json")) + parser.add_argument("--source-binding", type=Path, default=Path("ovk-source-binding.json")) + parser.add_argument("--diff", type=Path, default=Path(".ovk-pr.diff")) + parser.add_argument( + "--output", type=Path, + default=Path(os.environ["GITHUB_STEP_SUMMARY"]) if os.environ.get("GITHUB_STEP_SUMMARY") else None, + help="GitHub job summary path or another explicit Markdown file", + ) + parser.add_argument( + "--portable-output", type=Path, + help="Optional standalone, read-only scoped Markdown report for artifact reuse", + ) + args = parser.parse_args(argv) + if args.output is None: + print("no summary output path", file=sys.stderr) + return 2 + try: + if args.portable_output is not None: + # A report is an output, never an input or a replacement for the + # runner-owned job summary. Reject collisions before any deletion. + protected = (args.evidence, args.source_binding, args.diff, args.output) + if args.portable_output.resolve() in {p.resolve() for p in protected}: + raise ObservationError("portable output aliases source material or job summary") + if args.portable_output.is_symlink() or args.portable_output.is_dir(): + raise ObservationError("portable output must be an ordinary file path") + # If validation fails, a prior report must not survive to the + # always-running artifact uploader as if it belonged to this run. + args.portable_output.unlink(missing_ok=True) + evidence, evidence_bytes = _json_object(args.evidence) + receipt, _ = _json_object(args.source_binding) + diff_bytes = args.diff.read_bytes() + summary = build_summary(evidence, receipt, evidence_bytes, diff_bytes) + if args.portable_output is not None: + args.portable_output.write_text(summary + "\n", encoding="utf-8", newline="\n") + # The host owns GITHUB_STEP_SUMMARY; append so this Action composes + # with earlier steps and multiple invocations without clobbering them. + with args.output.open("a", encoding="utf-8", newline="\n") as output: + output.write(summary) + output.write("\n") + except (ObservationError, OSError, ValueError, TypeError) as exc: + print(f"OVK refusing unsupported job summary: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_action_combined_identity_contract.py b/tests/test_action_combined_identity_contract.py new file mode 100644 index 000000000..e49a592db --- /dev/null +++ b/tests/test_action_combined_identity_contract.py @@ -0,0 +1,71 @@ +"""Regression contract for combined Action version and source identity. + +These assertions cover the ordering and fail-closed composition of the +independently tested installer and source-binding helper. They do not establish +GitHub event authenticity, source-to-model fidelity, or relying authority. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def _steps() -> list[dict]: + """Load exactly the composite steps delivered to downstream Action users.""" + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + assert action["runs"]["using"] == "composite" + return action["runs"]["steps"] + + +def test_version_identity_precedes_verified_source_binding() -> None: + """Failure to resolve an explicitly requested version must stop the Action.""" + steps = _steps() + names = [step.get("name") for step in steps] + for required in ("Install OVK package", "Initialize verification directory if needed", + "Prepare change input", "Run OVK check path"): + assert names.count(required) == 1 + + install_i = names.index("Install OVK package") + init_i = names.index("Initialize verification directory if needed") + prepare_i = names.index("Prepare change input") + check_i = names.index("Run OVK check path") + assert install_i < init_i < prepare_i < check_i + + install = steps[install_i] + assert install["shell"] == "bash" + assert "set -euo pipefail" in install["run"] + assert 'pip install "open-verification-kernel==${OVK_PACKAGE_VERSION}"' in install["run"] + assert '|| install_from_checkout' not in install["run"] + assert "install_from_checkout" in install["run"] + + prepare = steps[prepare_i] + assert "set -euo pipefail" in prepare["run"] + assert "scripts/prepare_verified_change_input.py" in prepare["run"] + assert prepare["env"]["OVK_GITHUB_SHA"] == "${{ github.sha }}" + assert prepare["env"]["OVK_EVENT_HEAD_SHA"] == "${{ github.event.pull_request.head.sha }}" + assert prepare["env"]["OVK_EVENT_BASE_SHA"] == "${{ github.event.pull_request.base.sha }}" + assert "--github-sha" in prepare["run"] and "--base-sha" in prepare["run"] + assert "--pr-head-sha" in prepare["run"] + + +def test_unbound_manual_diff_cannot_be_published_by_default_check_path() -> None: + """An advisory caller-selected diff is explicitly outside the bound PR verdict.""" + steps = _steps() + by_name = {step["name"]: step for step in steps} + check = by_name["Emit OVK GitHub check (check path)"] + comment = by_name["Post OVK pull-request comment"] + assert "inputs.changed-files == ''" in check["if"] + assert "inputs.changed-files == ''" in comment["if"] + # Explicit manifests bypass source binding; public comments are forbidden. + assert "inputs.verification-manifest == ''" in comment["if"] + + upload = by_name["Upload OVK artifacts"] + assert "ovk-source-binding.json" in upload["with"]["path"] + # A receipt sidecar exists, but it remains outside the signed envelope. + source_doc = (ROOT / "docs/PR_SOURCE_BINDING_V1.md").read_text(encoding="utf-8") + assert "bound_into_signed_evidence_envelope=false" in source_doc diff --git a/tests/test_action_manifest_publication_guard.py b/tests/test_action_manifest_publication_guard.py new file mode 100644 index 000000000..b20890b12 --- /dev/null +++ b/tests/test_action_manifest_publication_guard.py @@ -0,0 +1,237 @@ +"""Manifest input has no authenticated PR-diff binding. + +The composite Action must not turn explicit manifest output into a GitHub +PR-wide Check or comment, nor inherit generated artifacts from an earlier +Action invocation in the same workspace. Local advisory verification remains. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def _steps() -> list[dict]: + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + return action["runs"]["steps"] + + +def test_stale_output_cleanup_precedes_package_install() -> None: + steps = _steps() + names = [step.get("name") for step in steps] + assert names.index("Invalidate stale generated Action outputs") < names.index("Install OVK package") + cleanup = next(step for step in steps if step.get("name") == "Invalidate stale generated Action outputs") + assert "set -euo pipefail" in cleanup["run"] + for artifact in ( + "ovk-source-binding.json", "ovk-evidence.json", "ovk-pr-comment.md", + "ovk-artifact-manifest.json", "ovk-attestation-envelope.json", + ".ovk-check-emitted", + ): + assert artifact in cleanup["run"] + + +def test_unbound_manifest_has_no_publication_path() -> None: + steps = _steps() + by_name = {step.get("name"): step for step in steps} + assert "Emit OVK GitHub check (verify path)" not in by_name + + default_check = by_name["Emit OVK GitHub check (check path)"] + assert "inputs.verification-manifest == ''" in default_check["if"] + assert "inputs.changed-files == ''" in default_check["if"] + + comment = by_name["Post OVK pull-request comment"] + assert "inputs.verification-manifest == ''" in comment["if"] + assert "inputs.changed-files == ''" in comment["if"] + assert "||" not in comment["if"] + + warning = by_name["Disclose unbound manifest inputs"] + assert "inputs.verification-manifest != ''" in warning["if"] + assert "source-to-PR binding" in warning["run"] + assert "strict enforcement are disabled" in warning["run"] + + +def test_all_caller_selected_inputs_degrade_action_level_decision() -> None: + step = next(s for s in _steps() if s.get("name") == "Set action outputs") + assert step["env"]["OVK_CHANGED_FILES"] == "${{ inputs.changed-files }}" + assert step["env"]["OVK_VERIFICATION_MANIFEST"] == "${{ inputs.verification-manifest }}" + assert 'or os.environ.get("OVK_CHANGED_FILES")' in step["run"] + assert 'decision_state = "unknown"' in step["run"] + assert 'recommendation = "require_human_review"' in step["run"] + + +def test_manifest_step_is_advisory_only_and_removes_stale_source_receipt() -> None: + step = next(s for s in _steps() if s.get("name") == "Run OVK multi-lane verification") + assert "inputs.verification-manifest != ''" in step["if"] + script = step["run"] + assert "set -euo pipefail" in script + assert 'if [ "$OVK_MODE" != "advisory" ]; then' in script + assert "exit 2" in script + for path in ( + "ovk-source-binding.json", + ".ovk-effective-input-path", + ".ovk-pr.diff", + ".ovk-check-emitted", + "ovk-evidence.json", + "ovk-pr-comment.md", + "ovk-attestation-envelope.json", + ): + assert path in script + assert "args+=(--advisory)" in script + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash step executes on Linux/macOS in this runtime test") +def test_strict_manifest_rejects_before_checker_or_artifact_reuse(tmp_path: Path) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + script = next( + s["run"] for s in _steps() + if s.get("name") == "Run OVK multi-lane verification" + ) + for name in ( + "ovk-source-binding.json", ".ovk-effective-input-path", + ".ovk-pr.diff", "ovk-evidence.json", ".ovk-check-emitted", + ): + (tmp_path / name).write_text("STALE", encoding="utf-8") + + env = dict(os.environ) + env.update({ + "OVK_MODE": "strict", + "OVK_VERIFICATION_MANIFEST": "fixtures/fake.json", + "OVK_BUNDLE_OUTPUT_DIR": "ovk-release-bundle", + "OVK_REPO": "owner/repo", + "OVK_HEAD_SHA": "a" * 40, + }) + result = subprocess.run( + ["bash", "-eo", "pipefail", "-c", script], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 2, result.stderr + assert "strict Action enforcement is refused" in result.stdout + for name in ( + "ovk-source-binding.json", ".ovk-effective-input-path", + ".ovk-pr.diff", "ovk-evidence.json", ".ovk-check-emitted", + ): + assert not (tmp_path / name).exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash runtime exercised on Linux/macOS") +def test_manifest_rejects_existing_nonempty_output_bundle(tmp_path: Path) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + step = next(step for step in _steps() if step.get("name") == "Run OVK multi-lane verification") + folder = tmp_path / "ovk-release-bundle" + folder.mkdir() + (folder / "old-ovk-evidence.json").write_text("STALE", encoding="utf-8") + env = dict(os.environ) + env.update({ + "OVK_MODE": "advisory", + "OVK_VERIFICATION_MANIFEST": "sample.json", + "OVK_BUNDLE_OUTPUT_DIR": str(folder), + "OVK_REPO": "owner/repo", + "OVK_HEAD_SHA": "b" * 40, + }) + proc = subprocess.run( + ["bash", "-eo", "pipefail", "-c", step["run"]], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 2, proc.stderr + assert "already contains files" in proc.stdout + assert (folder / "old-ovk-evidence.json").read_text(encoding="utf-8") == "STALE" + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash execution tested on Linux/macOS") +def test_manifest_pass_cannot_be_exported_as_pr_allow(tmp_path: Path) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + outputs = next(step for step in _steps() if step.get("name") == "Set action outputs") + folder = tmp_path / "ovk-bundle" + folder.mkdir() + (folder / "ovk-evidence.json").write_text( + '{"decision": {"decision_state": "allow", "merge_recommendation": "allow"}}', + encoding="utf-8", + ) + target = tmp_path / "action-output.txt" + env = dict(os.environ) + env.update({ + "OVK_VERIFICATION_MANIFEST": "selected.json", + "OVK_BUNDLE_OUTPUT_DIR": str(folder), + "OVK_EVIDENCE_OUTPUT": "ovk-evidence.json", + "GITHUB_OUTPUT": str(target), + }) + proc = subprocess.run( + ["bash", "-eo", "pipefail", "-c", outputs["run"]], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, proc.stderr + outcome = dict(line.split("=", 1) for line in target.read_text(encoding="utf-8").splitlines()) + assert outcome == { + "decision_state": "unknown", + "recommendation": "require_human_review", + "exit_code": "2", + "check_emitted": "false", + } + # The raw, explicitly scoped advisory observation remains available. + assert '"allow"' in (folder / "ovk-evidence.json").read_text(encoding="utf-8") + + +@pytest.mark.skipif(sys.platform == "win32", reason="Bash runtime exercised on Linux/macOS") +@pytest.mark.parametrize( + ("changed_files", "expected_state", "expected_exit"), + [("caller-selected.diff", "unknown", "2"), ("", "allow", "0")], +) +def test_caller_selected_diff_cannot_export_allow_but_default_bound_path_can( + tmp_path: Path, changed_files: str, expected_state: str, expected_exit: str, +) -> None: + if shutil.which("bash") is None: + pytest.skip("bash unavailable") + step = next(s for s in _steps() if s.get("name") == "Set action outputs") + (tmp_path / "ovk-evidence.json").write_text( + '{"decision": {"decision_state": "allow", "merge_recommendation": "allow"}}', + encoding="utf-8", + ) + target = tmp_path / "action-outputs.txt" + env = dict(os.environ) + env.update({ + "OVK_VERIFICATION_MANIFEST": "", + "OVK_CHANGED_FILES": changed_files, + "OVK_BUNDLE_OUTPUT_DIR": "ovk-release-bundle", + "OVK_EVIDENCE_OUTPUT": "ovk-evidence.json", + "GITHUB_OUTPUT": str(target), + }) + proc = subprocess.run( + ["bash", "-eo", "pipefail", "-c", step["run"]], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, proc.stderr + outcome = dict(line.split("=", 1) for line in target.read_text(encoding="utf-8").splitlines()) + assert outcome["decision_state"] == expected_state + assert outcome["exit_code"] == expected_exit + assert outcome["recommendation"] == ( + "allow" if not changed_files else "require_human_review" + ) + assert outcome["check_emitted"] == "false" diff --git a/tests/test_action_package_install_contract.py b/tests/test_action_package_install_contract.py new file mode 100644 index 000000000..80d3c7b90 --- /dev/null +++ b/tests/test_action_package_install_contract.py @@ -0,0 +1,127 @@ +"""Composite Action installation: exact version means no silent source fallback. + +Execute the actual Action run block in a sandbox with fake commands. No +package manager or network access is needed, and failed versions cannot be +mistaken for qualified development source installs. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def _action_install_script() -> str: + data = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + install = [item for item in data["runs"]["steps"] if item.get("name") == "Install OVK package"] + assert len(install) == 1 + return install[0]["run"] + + +def _exercise(tmp_path: Path, *, requested_version: str | None, fail_requested: bool = False): + # GitHub's Windows runners support Git Bash, whose PATH syntax differs + # from native Windows Python. BASH_ENV intercepts pip/python in Bash itself. + bash = shutil.which("bash") + if bash is None and os.name == "nt": + program_files = Path(os.environ.get("ProgramFiles", "C:/Program Files")) + for candidate in ( + program_files / "Git" / "bin" / "bash.exe", + program_files / "Git" / "usr" / "bin" / "bash.exe", + ): + if candidate.is_file(): + bash = str(candidate) + break + if bash is None: + pytest.skip("Bash unavailable for POSIX composite Action execution") + (tmp_path / "mock_commands.sh").write_text( + """pip() { + printf 'pip ' >> "$OVK_TEST_LOG" + printf '<%s>' "$@" >> "$OVK_TEST_LOG" + echo >> "$OVK_TEST_LOG" + if [ "$OVK_TEST_FAIL_VERSION" = "1" ] && + [ "$1" = "install" ] && + [ "$2" = "open-verification-kernel==$OVK_PACKAGE_VERSION" ]; then + return 47 + fi + return 0 +} +python() { + printf 'python ' >> "$OVK_TEST_LOG" + printf '<%s>' "$@" >> "$OVK_TEST_LOG" + echo >> "$OVK_TEST_LOG" + return 0 +} +""", + encoding="utf-8", + ) + log_path = tmp_path / "commands.log" + env = dict(os.environ) + env.update({ + "BASH_ENV": "./mock_commands.sh", + "OVK_TEST_LOG": "commands.log", + "OVK_TEST_FAIL_VERSION": "1" if fail_requested else "0", + }) + if requested_version is None: + env.pop("OVK_PACKAGE_VERSION", None) + else: + env["OVK_PACKAGE_VERSION"] = requested_version + result = subprocess.run( + [bash, "-c", _action_install_script()], + cwd=tmp_path, + env=env, + check=False, + text=True, + capture_output=True, + ) + return result, log_path.read_text(encoding="utf-8") + + +def test_requested_version_failed_resolution_does_not_install_from_checkout(tmp_path): + result, log = _exercise(tmp_path, requested_version="9.9.9", fail_requested=True) + assert result.returncode == 47 + assert "pip " in log + assert "sync_package_data.py" not in log + assert "pip <.>" not in log + assert "OVK installed distribution version" not in result.stdout + + +def test_requested_version_success_reports_dist_metadata_without_fallback(tmp_path): + result, log = _exercise(tmp_path, requested_version="1.2.1") + assert result.returncode == 0, result.stderr + assert "pip " in log + assert "python <-c>" in log + assert "sync_package_data.py" not in log + assert "pip <.>" not in log + assert "requested distribution version 1.2.1" in result.stdout + + +def test_unpinned_development_install_is_explicit_source_checkout(tmp_path): + result, log = _exercise(tmp_path, requested_version=None) + assert result.returncode == 0, result.stderr + assert "python " in log + assert "pip <.>" in log + assert "open-verification-kernel==" not in log + assert "package from Action source checkout" in result.stdout + + +def test_empty_version_uses_development_checkout(tmp_path): + result, log = _exercise(tmp_path, requested_version="") + assert result.returncode == 0, result.stderr + assert "pip <.>" in log + + +@pytest.mark.parametrize("supplied", ["1.2.1;touch should-not-exist", "1.2.1 $(echo injected)"]) +def test_version_is_one_quoted_package_argument(tmp_path, supplied): + result, log = _exercise(tmp_path, requested_version=supplied) + assert result.returncode == 0, result.stderr + assert "pip " in log + assert "sync_package_data.py" not in log + assert not (tmp_path / "should-not-exist").exists() diff --git a/tests/test_action_pr_source_binding_v1.py b/tests/test_action_pr_source_binding_v1.py new file mode 100644 index 000000000..1698297e0 --- /dev/null +++ b/tests/test_action_pr_source_binding_v1.py @@ -0,0 +1,226 @@ +"""Adversarial Git-object tests for source/diff binding in the Action. + +These are real Git commits and a two-parent merge-preview, not mocked SHA +strings. No network or GitHub credentials are involved. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from scripts import prepare_verified_change_input as binding + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="Git required") + + +def git(*args: str) -> str: + proc = subprocess.run( + ["git", *args], check=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, + ) + return proc.stdout.strip() + + +@pytest.fixture +def example_repo(tmp_path, monkeypatch): + repo = tmp_path / "consumer" + repo.mkdir() + monkeypatch.chdir(repo) + git("init", "-b", "main") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "OVK Test") + git("config", "core.autocrlf", "false") + (repo / "base.txt").write_text("base\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "base") + base = git("rev-parse", "HEAD") + git("checkout", "-b", "contributor") + (repo / "pr.txt").write_text("head\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "head") + pr_head = git("rev-parse", "HEAD") + git("checkout", "main") + git("merge", "--no-ff", "contributor", "-m", "merge-preview") + merge = git("rev-parse", "HEAD") + assert merge != pr_head + assert binding.parents() == [base, pr_head] + return {"repo": repo, "base": base, "pr_head": pr_head, "merge": merge} + + +def make_args(record): + return { + "event_name": "pull_request", + "base_sha": record["base"], + "pr_head_sha": record["pr_head"], + "github_sha": record["merge"], + } + + +def test_real_merge_preview_checks_the_tree_it_labels(example_repo): + path, diff, receipt = binding.prepare(**make_args(example_repo)) + expected = subprocess.run( + ["git", "diff", "--no-ext-diff", "--binary", + example_repo["base"], example_repo["merge"]], + check=True, stdout=subprocess.PIPE, + ).stdout + assert path == ".ovk-pr.diff" + assert diff == expected + assert b"pr.txt" in diff + assert receipt["event_pr_head_sha"] != receipt["evaluated_checkout_commit_sha"] + assert receipt["evidence_subject_sha"] == example_repo["merge"] + assert receipt["event_base_sha"] == example_repo["base"] + assert receipt["input_comparison"] == "event_base_to_verified_merge_preview" + assert receipt["input_diff_sha256"] == hashlib.sha256(diff).hexdigest() + assert receipt["input_structurally_bound_to_checkout"] is True + assert receipt["bound_into_signed_evidence_envelope"] is False + assert receipt["production_reliance_authorized"] is False + + +@pytest.mark.parametrize("mutation", ["head", "base", "checkout"]) +def test_identity_confusion_rejected(example_repo, mutation): + options = make_args(example_repo) + if mutation == "head": + options["pr_head_sha"] = example_repo["base"] + elif mutation == "base": + options["base_sha"] = example_repo["pr_head"] + else: + options["github_sha"] = example_repo["pr_head"] + with pytest.raises(binding.BindingError): + binding.prepare(**options) + + +def test_non_merge_pr_checkout_rejected(example_repo): + git("checkout", "contributor") + options = make_args(example_repo) + options["github_sha"] = example_repo["pr_head"] + with pytest.raises(binding.BindingError, match="merge parents"): + binding.prepare(**options) + + +@pytest.mark.parametrize("payload", [b"caller-selected input\n", b"caller-selected input\r\n"]) +def test_external_input_is_advisory_and_explicitly_unbound(example_repo, payload): + # Bytes must be stable across OSes: Windows text-mode writes translate LF to CRLF. + custom = example_repo["repo"] / "manual.diff" + custom.write_bytes(payload) + path, diff, receipt = binding.prepare( + **make_args(example_repo), changed_files=str(custom), + ) + assert path == str(custom) + assert diff == payload + assert receipt["input_diff_sha256"] == hashlib.sha256(payload).hexdigest() + assert receipt["input_comparison"] == "caller_supplied_unbound" + assert receipt["input_structurally_bound_to_checkout"] is False + with pytest.raises(binding.BindingError, match="cannot authorize strict"): + binding.prepare(**make_args(example_repo), changed_files=str(custom), mode="strict") + + +def test_caller_input_does_not_hide_mismatched_checkout(example_repo): + custom = example_repo["repo"] / "manual.diff" + custom.write_text("caller-selected input", encoding="utf-8") + options = make_args(example_repo) + options["github_sha"] = example_repo["base"] + with pytest.raises(binding.BindingError): + binding.prepare(**options, changed_files=str(custom)) + + +def test_privileged_event_refused(example_repo): + with pytest.raises(binding.BindingError, match="pull_request_target"): + binding.prepare( + event_name="pull_request_target", github_sha=example_repo["merge"], + ) + + +def test_bad_sha_and_non_pr_pr_metadata_fail_closed(example_repo): + with pytest.raises(binding.BindingError): + binding.prepare(event_name="pull_request", github_sha="HEAD") + with pytest.raises(binding.BindingError, match="non-PR"): + binding.prepare( + event_name="push", github_sha=example_repo["merge"], + pr_head_sha=example_repo["pr_head"], + ) + + +def test_cli_generates_separate_observational_receipt(example_repo): + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["merge"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + ]) == 0 + path = Path(".ovk-effective-input-path").read_text(encoding="utf-8").strip() + receipt = json.loads(Path("ovk-source-binding.json").read_text(encoding="utf-8")) + assert path == ".ovk-pr.diff" + # Bash command substitution strips LF, not CR. This must be exact on Windows. + assert Path(".ovk-effective-input-path").read_bytes() == b".ovk-pr.diff\n" + assert receipt["input_diff_sha256"] == hashlib.sha256( + Path(path).read_bytes() + ).hexdigest() + assert receipt["bound_into_signed_evidence_envelope"] is False + + +def test_source_binding_failure_emits_no_misleading_new_receipt(example_repo): + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["base"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + ]) == 2 + assert not Path("ovk-source-binding.json").exists() + + +def test_rejected_source_binding_discards_previous_generated_outputs(example_repo): + # A failed PR identity check must not leave old handoff/receipt files that + # the always-running artifact upload step could mistake for current output. + Path(".ovk-effective-input-path").write_bytes(b"stale.diff\\n") + Path("ovk-source-binding.json").write_bytes(b'{"forged": true}\\n') + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["base"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + ]) == 2 + assert not Path(".ovk-effective-input-path").exists() + assert not Path("ovk-source-binding.json").exists() + + +def test_caller_input_cannot_alias_generated_receipt(example_repo): + content = b"caller supplied input" + Path("ovk-source-binding.json").write_bytes(content) + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["merge"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + "--changed-files", "ovk-source-binding.json", + ]) == 2 + # Reserved generated artifacts must be removed even when used as inputs. + assert not Path("ovk-source-binding.json").exists() + + +def test_identical_generated_output_paths_fail_closed(example_repo): + reserved = Path("shared-receipt.json") + reserved.write_bytes(b"old generated data") + assert binding.main([ + "--event-name", "pull_request", + "--github-sha", example_repo["merge"], + "--base-sha", example_repo["base"], + "--pr-head-sha", example_repo["pr_head"], + "--input-path-output", str(reserved), + "--receipt-output", str(reserved), + ]) == 2 + assert not reserved.exists() + + +def test_non_pr_push_uses_actual_checkout_parent(example_repo): + _, diff, receipt = binding.prepare( + event_name="push", github_sha=example_repo["merge"], + ) + assert receipt["input_comparison"] == "checked_out_first_parent_to_head" + assert receipt["input_structurally_bound_to_checkout"] is True + assert b"pr.txt" in diff diff --git a/tests/test_action_source_binding_workflow_contract.py b/tests/test_action_source_binding_workflow_contract.py new file mode 100644 index 000000000..5243342ec --- /dev/null +++ b/tests/test_action_source_binding_workflow_contract.py @@ -0,0 +1,51 @@ +"""Assert the composite Action and CI invoke the bounded source identity contract.""" +from pathlib import Path +import yaml +ROOT = Path(__file__).resolve().parents[1] + +def test_action_executes_identity_helper_and_uploads_receipt(): + action = yaml.safe_load((ROOT / "action.yml").read_text()) + steps = action["runs"]["steps"] + prepare = [s for s in steps if s.get("name") == "Prepare change input"] + assert len(prepare) == 1 + script = prepare[0]["run"] + assert "scripts/prepare_verified_change_input.py" in script + assert "set -euo pipefail" in script + assert "--github-sha" in script and "--pr-head-sha" in script + assert "--base-sha" in script and "--mode" in script + assert "|| true" not in script + upload = next(s for s in steps if s.get("name") == "Upload OVK artifacts") + assert "ovk-source-binding.json" in upload["with"]["path"] + +def test_dogfood_uses_merge_preview_and_rejects_unbound_strict(): + workflow = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text()) + jobs = workflow["jobs"] + for name in ("action_default_diff_dogfood", "action_strict_dogfood"): + steps = jobs[name]["steps"] + checkout = next(s for s in steps if "actions/checkout@" in s.get("uses", "")) + assert checkout["with"]["ref"] == "${{ github.sha }}" + assert checkout["with"]["fetch-depth"] == 2 + assert checkout["with"]["persist-credentials"] is False + strict = jobs["action_strict_dogfood"] + assert strict["permissions"] == {"contents": "read"} + candidate = next(s for s in strict["steps"] if s.get("name", "").startswith("Strict Action")) + assert candidate["with"]["mode"] == "strict" + assert candidate["with"]["changed-files"] + assert candidate["with"]["emit-check"] == "false" + assert "test ! -f ovk-evidence.json" in strict["steps"][-1]["run"] + +def test_docs_reject_unproved_signed_source_binding(): + docs = (ROOT / "docs/PR_SOURCE_BINDING_V1.md").read_text() + assert "not yet incorporated" in docs + assert "production_reliance_authorized=false" in docs + +def test_unbound_caller_diff_never_emits_a_shareable_action_check_or_comment(): + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] + check = next(s for s in steps if s.get("name") == "Emit OVK GitHub check (check path)") + comment = next(s for s in steps if s.get("name") == "Post OVK pull-request comment") + warning = next(s for s in steps if s.get("name") == "Disclose unbound caller-selected inputs") + assert "inputs.changed-files == ''" in check["if"] + assert "inputs.changed-files == ''" in comment["if"] + assert "caller-supplied" in warning["run"] + assert "inputs.changed-files != ''" in warning["if"] diff --git a/tests/test_adoption_documentation_contract.py b/tests/test_adoption_documentation_contract.py new file mode 100644 index 000000000..aa85a36ca --- /dev/null +++ b/tests/test_adoption_documentation_contract.py @@ -0,0 +1,107 @@ +"""Public onboarding must not advertise an unpublished or unauthenticated release. + +The Git tag, Action source and Python distribution are distinct objects; +these tests guard copyable guidance, not the live publication authority. +""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HISTORICAL_SOURCE = "a27d5720f4350c00bca34f71d991c31f5a2f38c7" +UNPUBLISHED_TAG = "fraware/open-verification-kernel@v1.3.0-rc.1" +EXAMPLES = [ + ROOT / "examples/github_workflows/pilot_fork_adopter.yml", + ROOT / "examples/github_workflows/pilot_advisory_with_comment.yml", + ROOT / "examples/github_workflows/external_consumer.yml", +] + + +def test_public_readme_marks_published_source_and_legacy_fallback() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + assert HISTORICAL_SOURCE in readme + assert 'OVK_PACKAGE_VERSION: "1.2.1"' in readme + assert "v1.3.0-rc.1" in readme + assert "not published or tagged" in readme + assert "Action falls back to its checkout" in readme + assert "fail-closed installer" in readme + assert UNPUBLISHED_TAG not in readme + + +def test_active_pilot_workflows_do_not_install_nonexistent_candidate() -> None: + for path in EXAMPLES: + text = path.read_text(encoding="utf-8") + assert UNPUBLISHED_TAG not in text, path + assert 'OVK_PACKAGE_VERSION: "1.2.1"' in text, path + assert f"fraware/open-verification-kernel@{HISTORICAL_SOURCE}" in text, path + assert ( + "fallback to checkout" in text or "fall back to the Action checkout" in text + ), path + + +def test_fork_example_does_not_request_write_permissions_or_comment() -> None: + text = EXAMPLES[0].read_text(encoding="utf-8") + assert "pull-requests: write" not in text + assert "checks: write" not in text + assert 'post-comment: "true"' not in text + + +def test_integration_guide_separates_legacy_action_and_future_release() -> None: + text = (ROOT / "docs/INTEGRATION.md").read_text(encoding="utf-8") + assert "historical Action" in text + assert "not currently available" in text + assert HISTORICAL_SOURCE in text + assert UNPUBLISHED_TAG not in text + + +def test_full_license_and_notice_are_included_in_sdist() -> None: + license_text = (ROOT / "LICENSE").read_text(encoding="utf-8") + assert "Apache License" in license_text + assert "TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION" in license_text + assert "END OF TERMS AND CONDITIONS" in license_text + assert "Copyright 2026 fraware" in (ROOT / "NOTICE").read_text(encoding="utf-8") + assert "include NOTICE" in (ROOT / "MANIFEST.in").read_text(encoding="utf-8") + + +def test_preview_checklist_and_current_status_do_not_grant_release_authority() -> None: + checklist = (ROOT / "docs/DEVELOPER_PREVIEW_READINESS.md").read_text(encoding="utf-8") + status = (ROOT / "docs/CURRENT_RELEASE_STATUS.md").read_text(encoding="utf-8") + assert "NOT LAUNCH-AUTHORIZED" in checklist + assert "not exist as a public tag" in checklist + assert "unsigned" in status + assert "immutable=false" in status + assert "protected=false" in status + assert "issue #25" in status + + +def test_no_error_swallowing_for_requested_action_package_version() -> None: + action = (ROOT / "action.yml").read_text(encoding="utf-8") + requested = 'pip install "open-verification-kernel==$' + '{OVK_PACKAGE_VERSION}"' + assert requested in action + assert requested + " ||" not in action + assert 'echo "OVK install: requested distribution version' in action + + +def test_copyable_external_consumer_is_advisory_and_least_privilege() -> None: + text = (ROOT / "examples/github_workflows/external_consumer.yml").read_text( + encoding="utf-8" + ) + assert "mode: advisory" in text + assert "mode: strict" not in text + assert "contents: read" in text + assert "checks: write" not in text + assert "pull-requests: write" not in text + assert 'emit-check: "false"' in text + assert 'post-comment: "false"' in text + assert "changed-files:" not in text + + +def test_strict_required_check_reference_not_copyable_as_unpublished_rc() -> None: + text = ( + ROOT / "examples/github_workflows/branch_protection_required_check.yml" + ).read_text(encoding="utf-8") + assert UNPUBLISHED_TAG not in text + assert "REPLACE_WITH_AUTHORIZED_VERSION" in text + assert "REPLACE_WITH_VERIFIED_SHA" in text + assert "NOT INSTALLABLE UNTIL REPLACED" in text diff --git a/tests/test_collect_pilot_metrics.py b/tests/test_collect_pilot_metrics.py index 55e551921..559e2be84 100644 --- a/tests/test_collect_pilot_metrics.py +++ b/tests/test_collect_pilot_metrics.py @@ -16,7 +16,27 @@ def test_collect_pilot_metrics_schema_valid() -> None: require_schema_valid(metrics, schema, context="pilot metrics") validate_metrics(metrics) assert metrics["pilot_report"]["manifests_passed"] == metrics["pilot_report"]["manifests_total"] - assert metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] == 0.0 + assert metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] is None + + +def test_checker_block_on_fixture_does_not_qualify_false_positive_rate(tmp_path: Path) -> None: + """A failed fixture and a BLOCK are both tool-relative signals, not ground truth.""" + report = tmp_path / "pilot-report.json" + report.write_text( + json.dumps({ + "results": [ + {"passed": False, "merge_recommendation": "block", "elapsed_ms": 12.0}, + {"passed": True, "merge_recommendation": "allow", "elapsed_ms": 14.0}, + ], + "manifests_total": 2, + "manifests_passed": 1, + }), + encoding="utf-8", + ) + metrics = collect_pilot_metrics(source="pilot-dogfood", pilot_report_path=report) + validate_metrics(metrics) + assert metrics["adoption"]["pilot_dogfood"]["manifests_total"] == 2 + assert metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] is None def test_collect_pilot_metrics_with_evidence(tmp_path: Path) -> None: diff --git a/tests/test_ingest_external_pilot_metrics.py b/tests/test_ingest_external_pilot_metrics.py index 0544086d0..35ea8f99c 100644 --- a/tests/test_ingest_external_pilot_metrics.py +++ b/tests/test_ingest_external_pilot_metrics.py @@ -10,6 +10,7 @@ from ovk.core.json_io import read_json_file, write_json_file from ovk.core.schema_validation import require_schema_valid from scripts.ingest_external_pilot_metrics import ( + _metrics_from_artifacts, build_pilot_row, load_registry, upsert_pilot_row, @@ -39,11 +40,20 @@ def test_build_pilot_row_from_artifacts(tmp_path: Path) -> None: encoding="utf-8", ) - row = build_pilot_row("example/oss-pilot", artifacts_dir=artifacts_dir, ovk_version="1.2.0") + # The self-report must be separately selected as an adjudication source; + # merely placing it inside downloaded CI artifacts is insufficient. + row = build_pilot_row( + "example/oss-pilot", + artifacts_dir=artifacts_dir, + report_path=artifacts_dir / "external_pilot_report.json", + ovk_version="1.2.0", + ) assert row["repository"] == "example/oss-pilot" assert row["prs_evaluated"] == 4 assert row["false_positive_rate"] == 0.0 - assert row["median_check_latency_ms"] == 88.5 + # The explicit report changes the cohort count without a bound evidence ID. + # Preserve the reviewed report values, not latency from an unlinked artifact. + assert row["median_check_latency_ms"] is None assert row["status"] == "advisory" @@ -153,3 +163,307 @@ def test_validate_registry_rejects_invalid_row(tmp_path: Path) -> None: } with pytest.raises(ValueError, match="external pilots registry failed schema validation"): validate_registry(registry) + + +def test_checker_decisions_cannot_self_adjudicate_false_positives( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A blocked result is not independent evidence of a false alarm.""" + raw = tmp_path / "pilot-report.json" + raw.write_text( + json.dumps({ + "results": [ + {"passed": False, "merge_recommendation": "block", "elapsed_ms":12}, + {"passed": True, "merge_recommendation": "allow", "elapsed_ms":14}, + ] + }), + encoding="utf-8", + ) + monkeypatch.setattr( + "scripts.collect_pilot_metrics.discover_artifacts_dir", + lambda _: (raw, None, None), + ) + metrics = _metrics_from_artifacts(tmp_path) + assert metrics["prs_evaluated"] == 2 + assert metrics["prs_blocked"] == 1 + assert metrics["median_check_latency_ms"] == 13.0 + assert "false_positives" not in metrics + assert "false_positive_rate" not in metrics + + +def test_new_artifacts_clear_stale_external_error_labels( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Old reviewer labels must not be reused for a new unreviewed cohort.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: {"prs_evaluated": 7, "prs_blocked": 2}, + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row( + "example/oss-pilot", existing=existing, artifacts_dir=tmp_path + ) + assert row["prs_evaluated"] == 7 + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + + +@pytest.mark.parametrize( + "new_artifact_metrics", + [{"prs_blocked": 1}, {"median_check_latency_ms": 23.0}, {}], +) +def test_partial_new_artifacts_invalidate_stale_error_labels( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, new_artifact_metrics: dict +) -> None: + """No denominator or aggregate results must not imply zero new errors.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: new_artifact_metrics, + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, artifacts_dir=tmp_path) + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + + +def test_without_new_artifacts_historical_labels_are_not_silently_rewritten() -> None: + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 1, + "false_positive_rate": 0.5, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing) + assert row["false_positives"] == 1 + assert row["false_positive_rate"] == 0.5 + + +@pytest.mark.parametrize("input_kind", ["artifacts", "report"]) +def test_explicit_missing_pilot_input_fails_closed( + tmp_path: Path, input_kind: str +) -> None: + """A missing operator-selected input must never preserve prior metrics.""" + missing = tmp_path / ("missing-artifacts" if input_kind == "artifacts" else "missing.json") + params = {"artifacts_dir": missing} if input_kind == "artifacts" else {"report_path": missing} + with pytest.raises(FileNotFoundError, match="not found"): + build_pilot_row("example/oss-pilot", **params) + + +@pytest.mark.parametrize( + "new_artifact_metrics", + [{}, {"prs_blocked": 2}, {"median_check_latency_ms": 17.0}], +) +def test_partial_new_cohort_discards_all_stale_counts( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, new_artifact_metrics: dict +) -> None: + """A new partial cohort cannot inherit old PR totals or timings.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: new_artifact_metrics, + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "prs_evaluated": 100, + "prs_blocked": 20, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 5.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, artifacts_dir=tmp_path) + for key in ("prs_evaluated", "prs_blocked", "false_positives", "false_positive_rate", "median_check_latency_ms"): + assert row[key] == new_artifact_metrics.get(key), key + + +def test_new_report_cohort_without_error_labels_invalidates_old_rates( + tmp_path: Path, +) -> None: + """Fresh selected counts must not inherit unrelated older adjudications.""" + selected = tmp_path / "reviewer-report.json" + selected.write_text( + json.dumps({"repository": "example/oss-pilot", "prs_evaluated": 6}), + encoding="utf-8", + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "prs_evaluated": 4, + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, report_path=selected) + assert row["prs_evaluated"] == 6 + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + + +def test_new_selected_report_counts_clear_stale_latency_and_block_count( + tmp_path: Path, +) -> None: + selected = tmp_path / "new-cohort.json" + selected.write_text( + json.dumps({"repository": "example/oss-pilot", "prs_evaluated": 6}), + encoding="utf-8", + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "prs_evaluated": 4, + "prs_blocked": 2, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 71.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, report_path=selected) + assert row["prs_evaluated"] == 6 + for metric in ("prs_blocked", "false_positives", "false_positive_rate", "median_check_latency_ms"): + assert row[metric] is None + + +@pytest.mark.parametrize( + ("selected_fields", "stale_field"), + [ + ({"false_positives": 2}, "false_positive_rate"), + ({"false_positive_rate": 0.2}, "false_positives"), + ], +) +def test_partial_adjudication_does_not_retain_old_counterpart( + tmp_path: Path, selected_fields: dict, stale_field: str, +) -> None: + selected = tmp_path / "partial-review.json" + selected.write_text( + json.dumps({"repository": "example/oss-pilot", **selected_fields}), + encoding="utf-8", + ) + existing = { + "repository": "example/oss-pilot", + "status": "advisory", + "check_types": ["ci_secrets"], + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": False, + "ovk_version_pin": "1.2.1", + } + row = build_pilot_row("example/oss-pilot", existing=existing, report_path=selected) + assert row[stale_field] is None + for key, value in selected_fields.items(): + assert row[key] == value + + +def test_external_report_supplies_separate_reviewed_labels( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Explicit reviewer-provided labels are retained, not inferred.""" + monkeypatch.setattr( + "scripts.ingest_external_pilot_metrics._metrics_from_artifacts", + lambda _: {"prs_evaluated": 4, "prs_blocked": 2}, + ) + report = tmp_path / "reviewed.json" + report.write_text( + json.dumps({ + "repository": "example/oss-pilot", + "false_positives": 1, + "false_positive_rate": 0.25, + "notes": "Separately adjudicated by an external reviewer.", + }), + encoding="utf-8", + ) + row = build_pilot_row( + "example/oss-pilot", artifacts_dir=tmp_path, report_path=report + ) + assert row["prs_evaluated"] == 4 + assert row["false_positives"] == 1 + assert row["false_positive_rate"] == 0.25 + + +def test_embedded_artifact_self_report_cannot_set_error_or_strictness( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Embedded reports are not an approved independent source of labels.""" + pilot = tmp_path / "pilot-report.json" + pilot.write_text( + json.dumps({"results": [{"passed": True, "merge_recommendation": "allow", "elapsed_ms": 12}]}), + encoding="utf-8", + ) + embedded = tmp_path / "external_pilot_report.json" + embedded.write_text( + json.dumps({ + "repository": "attacker/redirected", + "prs_evaluated": 1000, + "false_positives": 0, + "false_positive_rate": 0.0, + "strict_enabled": True, + "notes": "unreviewed data in artifact bundle", + }), + encoding="utf-8", + ) + monkeypatch.setattr( + "scripts.collect_pilot_metrics.discover_artifacts_dir", + lambda _: (pilot, None, None), + ) + row = build_pilot_row("example/oss-pilot", artifacts_dir=tmp_path) + assert row["repository"] == "example/oss-pilot" + assert row["prs_evaluated"] == 1 + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + assert row["strict_enabled"] is False + assert row["status"] == "advisory" + + +def test_explicit_report_must_not_redirect_registry_identity(tmp_path: Path) -> None: + """Selected reporter may provide observations, not reassign the subject.""" + report = tmp_path / "report.json" + report.write_text( + json.dumps({ + "repository": "other/tenant", + "false_positives": 0, + "false_positive_rate": 0.0, + }), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="repository mismatch"): + build_pilot_row("example/oss-pilot", report_path=report) + + +def test_explicit_report_with_matching_repository_still_supported(tmp_path: Path) -> None: + report = tmp_path / "report.json" + report.write_text( + json.dumps({ + "repository": "example/oss-pilot", + "false_positives": 1, + "false_positive_rate": 0.25, + }), + encoding="utf-8", + ) + row = build_pilot_row("example/oss-pilot", report_path=report) + assert row["repository"] == "example/oss-pilot" + assert row["false_positives"] == 1 + assert row["false_positive_rate"] == 0.25 diff --git a/tests/test_pilot_release_integration_contract.py b/tests/test_pilot_release_integration_contract.py new file mode 100644 index 000000000..cdb858b28 --- /dev/null +++ b/tests/test_pilot_release_integration_contract.py @@ -0,0 +1,127 @@ +"""Cross-PR contract for a non-blocking, attributable pilot entry point. + +This guards copyable workflow safety and reporting provenance. It is not an +independent assessment of the pilot's scientific accuracy or adopter identity. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +HISTORICAL_ACTION = "fraware/open-verification-kernel@a27d5720f4350c00bca34f71d991c31f5a2f38c7" + + +def test_fork_adopter_example_has_no_publication_or_write_token() -> None: + workflow_path = ROOT / "examples/github_workflows/pilot_fork_adopter.yml" + raw = workflow_path.read_text(encoding="utf-8") + workflow = yaml.safe_load(raw) + permissions = workflow["permissions"] + assert permissions["contents"] == "read" + assert "checks" not in permissions + assert permissions.get("pull-requests", "read") == "read" + + steps = workflow["jobs"]["ovk-ci-secrets-pilot"]["steps"] + checkouts = [s for s in steps if s.get("uses", "").startswith("actions/checkout@")] + assert len(checkouts) == 1 + assert checkouts[0]["with"]["persist-credentials"] is False + assert re.fullmatch(r"actions/checkout@[0-9a-f]{40}", checkouts[0]["uses"]) + + ovk_steps = [s for s in steps if s.get("uses", "").startswith("fraware/open-verification-kernel@")] + assert len(ovk_steps) == 2 + for step in ovk_steps: + assert step["uses"] == HISTORICAL_ACTION + assert step["with"]["mode"] == "advisory" + assert step["with"]["emit-check"] == "false" + assert step["with"]["post-comment"] == "false" + + assert ovk_steps[0]["with"]["changed-files"] == "ovk-pr.diff" + assert "unbound source input" in (ROOT / "docs/EXTERNAL_PILOT_PLAYBOOK.md").read_text(encoding="utf-8") + assert "external_pilot_report.json" not in raw + + +def test_copyable_readme_example_is_least_privilege() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + section = readme.split("## GitHub Actions", 1)[1].split("## Built for agents", 1)[0] + assert "contents: read" in section + assert "checks: write" not in section + assert "pull-requests: write" not in section + assert HISTORICAL_ACTION in section + assert 'emit-check: "false"' in section + assert 'post-comment: "false"' in section + assert "falls back to its checkout" in section + + +def test_pilot_template_never_invents_measured_zeroes() -> None: + report = json.loads( + (ROOT / "docs/templates/external_pilot_report.template.json").read_text(encoding="utf-8") + ) + for field in ( + "advisory_start", "advisory_end", "prs_evaluated", "prs_blocked", + "false_positives", "false_positive_rate", "median_check_latency_ms", + ): + assert report[field] is None, field + assert report["strict_enabled"] is False + + +def test_release_docs_and_strict_example_are_explicitly_qualified() -> None: + status = (ROOT / "docs/CURRENT_RELEASE_STATUS.md").read_text(encoding="utf-8") + integration = (ROOT / "docs/INTEGRATION.md").read_text(encoding="utf-8") + assert "protected=false" in status + assert "verification.verified=false" in status + assert "immutable=false" in status + assert "v1.3.0-rc.1" in status + assert "shared HMAC secret" in integration + assert "No currently approved copyable workflow" in integration + + +def test_published_fixture_error_rates_are_unadjudicated_unknown() -> None: + """Corrected archived aggregates must not advertise 0% without ground truth.""" + registry = json.loads( + (ROOT / "docs/benchmarks/external-pilots-registry.json").read_text(encoding="utf-8") + ) + summary = json.loads( + (ROOT / "docs/benchmarks/adoption-summary.json").read_text(encoding="utf-8") + ) + assert summary["pilot_dogfood"]["false_positive_rate"] is None + + for data in (registry, summary): + maintained = [ + row for row in data["external_pilots"] + if "Maintained consumer" in row.get("notes", "") + or "In-repo maintained" in row.get("notes", "") + ] + assert len(maintained) == 3 + for row in maintained: + assert row["false_positives"] is None + assert row["false_positive_rate"] is None + assert "not independently" in row["notes"] or "no independent" in row["notes"] + + study = (ROOT / "docs/PILOT_CASE_STUDIES.md").read_text(encoding="utf-8") + assert "0.0 (fixtures)" not in study + assert "Target: under 5%" not in study + assert "reported as unknown" in study + + +def test_historical_pilot_case_studies_disclose_unreviewed_fixture_rates() -> None: + """Archival per-pilot reports must not equate fixture outcomes with FPR.""" + paths = ( + "docs/pilots/fastapi-terraform/REPORT.md", + "docs/pilots/express-actions/REPORT.md", + "docs/pilots/infra-terraform-k8s/REPORT.md", + ) + for path in paths: + report = (ROOT / path).read_text(encoding="utf-8") + assert "historical" in report.lower() + assert "null/unknown" in report + assert "Independently adjudicated false positives | **Unknown**" in report + assert "Independently adjudicated false-positive rate | **Unknown**" in report + assert "Independently adjudicated false negatives | **Unknown**" in report + assert "| False positive rate | **0.0**" not in report + assert "FP under 5%" not in report + assert "immutable tag / audited SHA only" not in report diff --git a/tests/test_release_workflow_event_provenance.py b/tests/test_release_workflow_event_provenance.py index cf29a7f7b..440b505c8 100644 --- a/tests/test_release_workflow_event_provenance.py +++ b/tests/test_release_workflow_event_provenance.py @@ -31,7 +31,16 @@ def test_required_generic_validation_workflows_checkout_recorded_head() -> None: text = (REPO / relative).read_text(encoding="utf-8") checkout_count = text.count("uses: actions/checkout@") assert checkout_count >= 1, relative - assert text.count(exact_ref) == checkout_count, relative + if relative == ".github/workflows/ci.yml": + # Source-validation and release-gate jobs still checkout the + # PR's actual head. The two Action dogfood jobs intentionally + # exercise GitHub's evaluated synthetic merge-preview commit, + # whose SHA is github.sha for a pull_request event. + merge_ref = "ref: ${{ github.sha }}" + assert text.count(merge_ref) == 2, relative + assert text.count(exact_ref) == checkout_count - 2, relative + else: + assert text.count(exact_ref) == checkout_count, relative def test_release_collector_supports_dispatch_only_filter() -> None: diff --git a/tests/test_render_pilot_metrics.py b/tests/test_render_pilot_metrics.py index 7b48d3a81..c212bb686 100644 --- a/tests/test_render_pilot_metrics.py +++ b/tests/test_render_pilot_metrics.py @@ -28,6 +28,15 @@ def test_render_adoption_summary_shape() -> None: assert summary["pilot_dogfood"]["manifests_passed"] == metrics["adoption"]["pilot_dogfood"]["manifests_passed"] +def test_renderer_discards_unadjudicated_legacy_fixture_rate() -> None: + """A stale aggregate must not recreate a self-judged 0% error rate.""" + metrics = collect_pilot_metrics(source="local") + metrics["adoption"]["pilot_dogfood"]["false_positive_rate"] = 0.0 + rendered = render_adoption_summary(metrics) + validate_summary(rendered) + assert rendered["pilot_dogfood"]["false_positive_rate"] is None + + def test_adoption_summary_template_is_schema_valid() -> None: template = read_json_file(Path("docs/benchmarks/adoption-summary.json")) schema = read_json_file(Path("schemas/adoption.summary.schema.json")) diff --git a/tests/test_source_bound_merge_subject.py b/tests/test_source_bound_merge_subject.py new file mode 100644 index 000000000..f1a22d758 --- /dev/null +++ b/tests/test_source_bound_merge_subject.py @@ -0,0 +1,168 @@ +"""A real merge-preview must remain the evidence subject after PR event ingestion. + +Local source checks are structural, not signature verification or generalized +source-to-model fidelity. This test creates real Git commits and a merge. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from ovk.cli import app +from ovk.core.context import build_repository_context +from ovk.core.source_binding_identity import ( + SourceSubjectMismatch, require_bound_merge_preview_subject, +) +from scripts.prepare_verified_change_input import prepare + + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="Git required") + + +def git(*args: str) -> str: + result = subprocess.run( + ["git", *args], capture_output=True, text=True, check=True, + ) + return result.stdout.strip() + + +@pytest.fixture +def merge_pr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict: + monkeypatch.chdir(tmp_path) + git("init", "-b", "main") + git("config", "user.email", "test@example.invalid") + git("config", "user.name", "Source Identity Test") + git("config", "core.autocrlf", "false") + (tmp_path / "original.txt").write_text("original\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "base") + base = git("rev-parse", "HEAD") + git("checkout", "-b", "contribution") + (tmp_path / "example.py").write_text("sink()\n", encoding="utf-8") + git("add", ".") + git("commit", "-m", "head") + pr_head = git("rev-parse", "HEAD") + git("checkout", "main") + git("merge", "--no-ff", "contribution", "-m", "merge preview") + merged = git("rev-parse", "HEAD") + assert merged != pr_head + event_path = tmp_path / "pr-event.json" + event_path.write_text(json.dumps({ + "repository": {"full_name": "pilot/repo"}, + "pull_request": { + "number": 17, + "head": {"sha": pr_head}, + "base": {"sha": base}, + }, + "sender": {"login": "contributor", "type": "User"}, + }), encoding="utf-8") + path, diff, receipt = prepare( + event_name="pull_request", github_sha=merged, base_sha=base, pr_head_sha=pr_head, + ) + assert path == ".ovk-pr.diff" + diff_path = tmp_path / path + diff_path.write_bytes(diff) + receipt_path = tmp_path / "ovk-source-binding.json" + receipt_path.write_text(json.dumps(receipt), encoding="utf-8") + return { + "base": base, "head": pr_head, "merge": merged, + "event": event_path, "diff": diff_path, + "receipt": receipt_path, + } + + +def validate(case: dict) -> None: + require_bound_merge_preview_subject( + receipt_path=case["receipt"], + diff_path=case["diff"], + event_path=case["event"], + repo="pilot/repo", + head_sha=case["merge"], + base_sha=case["base"], + ) + + +def test_real_pr_subject_receipt_and_context_agree(merge_pr: dict) -> None: + validate(merge_pr) + ordinary = build_repository_context( + changed_files=["example.py"], github_event_path=merge_pr["event"], + repo="pilot/repo", head_sha=merge_pr["merge"], + ) + assert ordinary.head_sha == merge_pr["head"] + bound = build_repository_context( + changed_files=["example.py"], github_event_path=merge_pr["event"], + repo="pilot/repo", head_sha=merge_pr["merge"], + preserve_explicit_subject=True, + ) + assert bound.head_sha == merge_pr["merge"] + assert bound.base_sha == merge_pr["base"] + assert bound.branch_metadata["event_pr_head_sha"] == merge_pr["head"] + assert bound.branch_metadata["source_subject_origin"] == "locally_bound_merge_preview" + + +@pytest.mark.parametrize("mutation", ["receipt_head", "diff", "pr_event", "checkout", "requested_head", "base"]) +def test_mismatched_source_materials_fail_closed(merge_pr: dict, mutation: str) -> None: + case = dict(merge_pr) + if mutation == "receipt_head": + receipt = json.loads(case["receipt"].read_text()) + receipt["evidence_subject_sha"] = case["head"] + case["receipt"].write_text(json.dumps(receipt)) + elif mutation == "diff": + case["diff"].write_bytes(case["diff"].read_bytes() + b"tamper") + elif mutation == "pr_event": + event = json.loads(case["event"].read_text()) + event["pull_request"]["head"]["sha"] = case["base"] + case["event"].write_text(json.dumps(event)) + elif mutation == "checkout": + git("checkout", "contribution") + elif mutation == "requested_head": + case["merge"] = case["head"] + elif mutation == "base": + case["base"] = case["head"] + with pytest.raises(SourceSubjectMismatch): + validate(case) + + +def test_cli_emits_observed_merge_subject_and_keeps_event_pr_head(merge_pr: dict, tmp_path: Path) -> None: + output = tmp_path / "outputs" + outcome = CliRunner().invoke( + app, [ + "check", "--diff", str(merge_pr["diff"]), + "--github-event", str(merge_pr["event"]), + "--source-binding", str(merge_pr["receipt"]), + "--repo", "pilot/repo", + "--head-sha", merge_pr["merge"], + "--base-sha", merge_pr["base"], + "--output-dir", str(output), + "--advisory", "--no-cache", + ], + ) + assert outcome.exit_code == 0, outcome.output + bundle = json.loads((output / "ovk-evidence.json").read_text()) + assert bundle["subject"]["head_sha"] == merge_pr["merge"] + assert bundle["subject"]["base_sha"] == merge_pr["base"] + assert bundle["subject"]["repo"] == "pilot/repo" + + +def test_cli_rejects_fake_subject_before_publishing_evidence(merge_pr: dict, tmp_path: Path) -> None: + output = tmp_path / "out-rejected" + outcome = CliRunner().invoke( + app, [ + "check", "--diff", str(merge_pr["diff"]), + "--github-event", str(merge_pr["event"]), + "--source-binding", str(merge_pr["receipt"]), + "--repo", "pilot/repo", + "--head-sha", merge_pr["head"], + "--output-dir", str(output), + "--advisory", + ], + ) + assert outcome.exit_code != 0 + assert "refusing unbound evidence subject" in outcome.output + assert not (output / "ovk-evidence.json").exists() diff --git a/tests/test_source_scoped_job_summary.py b/tests/test_source_scoped_job_summary.py new file mode 100644 index 000000000..673aa1d5f --- /dev/null +++ b/tests/test_source_scoped_job_summary.py @@ -0,0 +1,271 @@ +"""Portable informational PR summary from exact local evidence and source receipt. + +These tests do not assert event authenticity, checker soundness, or permission +to merge. They guard a low-friction, least-privilege observation surface. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path + +import pytest +import yaml + +from scripts.render_source_bound_job_summary import ObservationError, build_summary, main + + +ROOT = Path(__file__).resolve().parents[1] +BASE = "a" * 40 +HEAD = "b" * 40 +MERGE = "c" * 40 +TREE = "f" * 40 +DIFF_BYTES = b"diff --git a/sample.py b/sample.py\n+authorized = True\n" +DIFF = hashlib.sha256(DIFF_BYTES).hexdigest() + + +def receipt() -> dict: + return { + "schema_version": "ovk.pr_source_input_binding.v1-development", + "event_name": "pull_request", + "event_base_sha": BASE, + "event_pr_head_sha": HEAD, + "evaluated_checkout_commit_sha": MERGE, + "evaluated_checkout_tree_sha": TREE, + "evidence_subject_sha": MERGE, + "input_comparison": "event_base_to_verified_merge_preview", + "input_diff_sha256": DIFF, + "input_structurally_bound_to_checkout": True, + "independent_event_authentication": False, + "bound_into_signed_evidence_envelope": False, + "production_reliance_authorized": False, + } + + +def evidence(state: str = "allow") -> dict: + return { + "schema_version": "ovk.bundle.v1", + "bundle_id": "local-case", + "subject": {"repo": "example/repo", "head_sha": MERGE}, + "evidence": [], + "decision": {"decision_state": state, "merge_recommendation": state}, + } + + +def test_shareable_summary_is_precisely_scoped_and_identifiable() -> None: + data = evidence("block") + raw = json.dumps(data, sort_keys=True).encode() + markdown = build_summary(data, receipt(), raw, DIFF_BYTES) + assert "advisory PR observation" in markdown + assert "**Recorded scoped decision:** `block`" in markdown + for id_ in (BASE, HEAD, MERGE, TREE, DIFF): + assert id_ in markdown + assert hashlib.sha256(raw).hexdigest() in markdown + assert "neither the event nor the evidence is independently authenticated" in markdown + assert "An `allow` is not permission to merge." in markdown + assert "not signed into the evidence envelope" in markdown + assert "supported-source completeness" in markdown + + +@pytest.mark.parametrize( + ("key", "replacement"), + [ + ("schema_version", "unknown.schema"), + ("event_name", "push"), + ("input_comparison", "caller_supplied_unbound"), + ("input_structurally_bound_to_checkout", False), + ("independent_event_authentication", True), + ("bound_into_signed_evidence_envelope", True), + ("production_reliance_authorized", True), + ("evaluated_checkout_commit_sha", "wrong"), + ("event_pr_head_sha", "missing"), + ("event_base_sha", "BAD"), + ("evaluated_checkout_tree_sha", "missing"), + ("input_diff_sha256", "not-a-digest"), + ("evidence_subject_sha", BASE), + ], +) +def test_unbound_unexpected_and_malformed_receipts_fail_closed(key: str, replacement: object) -> None: + altered = copy.deepcopy(receipt()) + altered[key] = replacement + with pytest.raises(ObservationError): + build_summary(evidence(), altered, b"{}", DIFF_BYTES) + + +def test_subject_mismatch_and_unknown_decision_fail_closed() -> None: + altered = evidence() + altered["subject"]["head_sha"] = HEAD + with pytest.raises(ObservationError, match="evidence subject"): + build_summary(altered, receipt(), b"{}", DIFF_BYTES) + with pytest.raises(ObservationError, match="decision state"): + build_summary(evidence("authoritative_verified"), receipt(), b"{}", DIFF_BYTES) + + +def test_renderer_appends_without_overwriting_existing_job_summary(tmp_path: Path) -> None: + observation = tmp_path / "evidence.json" + binding = tmp_path / "binding.json" + diff = tmp_path / "actual-pr.diff" + diff.write_bytes(DIFF_BYTES) + output = tmp_path / "summary.md" + observation.write_text(json.dumps(evidence()), encoding="utf-8") + binding.write_text(json.dumps(receipt()), encoding="utf-8") + output.write_text("## Existing reviewer summary\n\n", encoding="utf-8") + + assert main([ + "--evidence", str(observation), + "--source-binding", str(binding), + "--diff", str(diff), + "--output", str(output), + ]) == 0 + contents = output.read_text(encoding="utf-8") + assert contents.startswith("## Existing reviewer summary\n\n") + assert "advisory PR observation" in contents + assert MERGE in contents + assert observation.read_bytes() and binding.read_bytes() + + +def test_no_summary_written_from_unbound_input(tmp_path: Path) -> None: + observation = tmp_path / "evidence.json" + binding = tmp_path / "binding.json" + diff = tmp_path / "actual-pr.diff" + diff.write_bytes(DIFF_BYTES) + output = tmp_path / "summary.md" + observation.write_text(json.dumps(evidence("allow")), encoding="utf-8") + unbound = receipt() + unbound["input_structurally_bound_to_checkout"] = False + binding.write_text(json.dumps(unbound), encoding="utf-8") + + assert main([ + "--evidence", str(observation), + "--source-binding", str(binding), + "--diff", str(diff), + "--output", str(output), + ]) == 2 + assert not output.exists() + + +def test_job_summary_only_runs_for_default_source_bound_advisory_pr() -> None: + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] + step = next(s for s in steps if s.get("name") == "Share scoped advisory observation in job summary") + condition = step["if"] + for guard in ( + "github.event_name == 'pull_request'", + "inputs.mode == 'advisory'", + "inputs.verification-manifest == ''", + "inputs.changed-files == ''", + "inputs.use-check == 'true'", + ): + assert guard in condition + assert 'render_source_bound_job_summary.py' in step["run"] + assert "--diff .ovk-pr.diff" in step["run"] + assert '"$GITHUB_STEP_SUMMARY"' in step["run"] + assert "GITHUB_TOKEN" not in step.get("env", {}) + assert "GITHUB_TOKEN" not in step["run"] + assert "check-runs" not in step["run"] + + +def test_modified_diff_bytes_cannot_be_described_by_old_receipt() -> None: + with pytest.raises(ObservationError, match="diff bytes"): + build_summary(evidence(), receipt(), b"{}", DIFF_BYTES + b"unexpected") + + +def test_missing_source_diff_never_creates_report(tmp_path: Path) -> None: + observation = tmp_path / "evidence.json" + binding = tmp_path / "binding.json" + destination = tmp_path / "summary.md" + observation.write_text(json.dumps(evidence()), encoding="utf-8") + binding.write_text(json.dumps(receipt()), encoding="utf-8") + assert main([ + "--evidence", str(observation), + "--source-binding", str(binding), + "--diff", str(tmp_path / "absent.diff"), + "--output", str(destination), + ]) == 2 + assert not destination.exists() + + +def test_portable_report_matches_append_only_job_summary(tmp_path: Path) -> None: + evidence_path = tmp_path / "ovk-evidence.json" + receipt_path = tmp_path / "ovk-source-binding.json" + diff_path = tmp_path / ".ovk-pr.diff" + job_summary = tmp_path / "GITHUB_STEP_SUMMARY" + portable = tmp_path / "ovk-advisory-pr-summary.md" + evidence_path.write_text(json.dumps(evidence("allow")), encoding="utf-8") + receipt_path.write_text(json.dumps(receipt()), encoding="utf-8") + diff_path.write_bytes(DIFF_BYTES) + job_summary.write_text("## First tool\n\n", encoding="utf-8") + rc = main([ + "--evidence", str(evidence_path), "--source-binding", str(receipt_path), + "--diff", str(diff_path), "--output", str(job_summary), + "--portable-output", str(portable), + ]) + assert rc == 0 + expected = build_summary(evidence(), receipt(), evidence_path.read_bytes(), DIFF_BYTES) + assert portable.read_text(encoding="utf-8") == expected + "\n" + assert job_summary.read_text(encoding="utf-8") == "## First tool\n\n" + expected + "\n" + assert "An `allow` is not permission to merge" in portable.read_text(encoding="utf-8") + assert "independently authenticated" in portable.read_text(encoding="utf-8") + assert "ovk-advisory-pr-summary" not in evidence_path.read_text(encoding="utf-8") + + +def test_bad_source_does_not_retain_preexisting_shareable_report(tmp_path: Path) -> None: + evidence_path = tmp_path / "ovk-evidence.json" + receipt_path = tmp_path / "ovk-source-binding.json" + diff_path = tmp_path / ".ovk-pr.diff" + job_summary = tmp_path / "GITHUB_STEP_SUMMARY" + portable = tmp_path / "ovk-advisory-pr-summary.md" + evidence_path.write_text(json.dumps(evidence()), encoding="utf-8") + receipt_path.write_text(json.dumps(receipt()), encoding="utf-8") + diff_path.write_bytes(DIFF_BYTES + b"tampered") + portable.write_text("UNSAFE STALE ALLOW", encoding="utf-8") + assert main([ + "--evidence", str(evidence_path), "--source-binding", str(receipt_path), + "--diff", str(diff_path), "--output", str(job_summary), + "--portable-output", str(portable), + ]) == 2 + assert not portable.exists() + assert not job_summary.exists() + + +@pytest.mark.parametrize("alias", ["evidence", "receipt", "diff", "summary"]) +def test_portable_report_cannot_overwrite_verification_input( + tmp_path: Path, alias: str, +) -> None: + paths = { + "evidence": tmp_path / "evidence.json", + "receipt": tmp_path / "receipt.json", + "diff": tmp_path / "change.diff", + "summary": tmp_path / "job-summary.md", + } + paths["evidence"].write_text(json.dumps(evidence()), encoding="utf-8") + paths["receipt"].write_text(json.dumps(receipt()), encoding="utf-8") + paths["diff"].write_bytes(DIFF_BYTES) + before = paths[alias].read_bytes() if paths[alias].exists() else None + assert main([ + "--evidence", str(paths["evidence"]), + "--source-binding", str(paths["receipt"]), + "--diff", str(paths["diff"]), + "--output", str(paths["summary"]), + "--portable-output", str(paths[alias]), + ]) == 2 + after = paths[alias].read_bytes() if paths[alias].exists() else None + assert before == after + + +def test_action_uploads_scoped_report_without_uploading_raw_diff(tmp_path: Path) -> None: + action = yaml.safe_load((ROOT / "action.yml").read_text(encoding="utf-8")) + steps = action["runs"]["steps"] + summary = next(s for s in steps if s.get("name") == "Share scoped advisory observation in job summary") + assert "--portable-output ovk-advisory-pr-summary.md" in summary["run"] + assert "inputs.changed-files == ''" in summary["if"] + assert "inputs.verification-manifest == ''" in summary["if"] + upload = next(s for s in steps if s.get("name") == "Upload OVK artifacts") + assert "ovk-advisory-pr-summary.md" in upload["with"]["path"] + # A diff can contain credentials or private source. It is used and + # hashed locally, never included in portable uploaded artifacts. + assert ".ovk-pr.diff" not in upload["with"]["path"] + cleanup = next(s for s in steps if s.get("name") == "Invalidate stale generated Action outputs") + assert "ovk-advisory-pr-summary.md" in cleanup["run"]