From d548fad9ab0cc016444ea9834dcbf70542a47acb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 06:28:07 +0900 Subject: [PATCH 1/7] fix(strix): resolve evidence binder from trusted source --- scripts/ci/strix_quick_gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index c7d3667465..9eed5e2e71 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -243,7 +243,7 @@ PY sanitize_remediation_evidence_claims() { local log_file="$1" local report_root="$2" - local binder="$REPO_ROOT/scripts/ci/strix_evidence_binding.py" + local binder="$SCRIPT_DIR/strix_evidence_binding.py" local report_file if [ ! -f "$binder" ] || [ -L "$binder" ]; then From 2db8684be0439857b92adcc5497d88f0dcdf3961 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 06:27:58 +0900 Subject: [PATCH 2/7] test(strix): bind evidence helper to trusted source root --- scripts/ci/test_strix_quick_gate.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 150b9102b3..7a93cffbc7 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -492,6 +492,8 @@ assert_changed_file_membership_uses_cached_normalized_paths() { assert_strix_evidence_binding_contract() { assert_file_contains "$GATE_SCRIPT" "sanitize_remediation_evidence_claims" "strix gate sanitizes false already-applied remediation claims" assert_file_contains "$GATE_SCRIPT" 'scripts/ci/strix_evidence_binding.py' "strix gate binds remediation evidence through the tested Python binder" + assert_file_contains "$GATE_SCRIPT" 'local binder="$SCRIPT_DIR/strix_evidence_binding.py"' "strix gate resolves its trusted evidence binder from the central script directory" + assert_file_not_contains "$GATE_SCRIPT" 'local binder="$REPO_ROOT/scripts/ci/strix_evidence_binding.py"' "strix gate never resolves the trusted binder from the consumer repository root" assert_file_contains "$GATE_SCRIPT" "evidence_scope=pr_delta" "strix gate labels PR-delta findings with authenticated provenance" assert_file_contains "$GATE_SCRIPT" "evidence_scope=repository_baseline" "strix gate labels unchanged-path findings as repository_baseline" assert_file_contains "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" 'PR_DELTA = "pr_delta"' "strix evidence binder defines pr_delta scope" From 747fc3e182025a4e35e620da3d502381632664c3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 23 Sep 2026 02:17:13 +0900 Subject: [PATCH 3/7] fix(security): require exact-head release dependency evidence --- .../agent-review-runtime-quality-ci.yml | 3 + .github/workflows/dependency-review.yml | 56 ++++++- scripts/ci/release_dependency_evidence.py | 137 ++++++++++++++++++ scripts/ci/strix_quick_gate.sh | 12 +- ...dency_review_reusable_workflow_contract.py | 35 ++++- tests/test_release_dependency_evidence.py | 124 ++++++++++++++++ .../test_strix_mandatory_evidence_contract.py | 19 +++ 7 files changed, 369 insertions(+), 17 deletions(-) create mode 100644 scripts/ci/release_dependency_evidence.py create mode 100644 tests/test_release_dependency_evidence.py create mode 100644 tests/test_strix_mandatory_evidence_contract.py diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index a601e25522..82e1d60fa8 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -220,6 +220,7 @@ jobs: tests/test_docs_only_pr_runner_admission.py|\ tests/test_strix_changed_path_policy.py|\ tests/test_strix_evidence_binding.py|\ + tests/test_strix_mandatory_evidence_contract.py|\ tests/test_strix_model_behavior_error.py|\ tests/test_strix_nvidia_nim_not_found_fallback.py|\ tests/test_strix_workflow_dependency_hashes.py|\ @@ -401,6 +402,7 @@ jobs: tests/test_docs_only_pr_runner_admission.py \ tests/test_strix_changed_path_policy.py \ tests/test_strix_evidence_binding.py \ + tests/test_strix_mandatory_evidence_contract.py \ tests/test_strix_model_behavior_error.py \ tests/test_strix_nvidia_nim_not_found_fallback.py \ tests/test_strix_workflow_dependency_hashes.py \ @@ -410,6 +412,7 @@ jobs: scripts/ci/strix_evidence_binding.py \ tests/test_strix_changed_path_policy.py \ tests/test_strix_evidence_binding.py \ + tests/test_strix_mandatory_evidence_contract.py \ tests/test_strix_model_behavior_error.py \ tests/test_strix_nvidia_nim_not_found_fallback.py \ tests/test_strix_workflow_dependency_hashes.py \ diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 1bb83c2baf..c34fd5a6d4 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -104,6 +104,16 @@ jobs: with: persist-credentials: false + - name: Materialize exact trusted release-policy verifier + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ContextualWisdomLab/.github + ref: ${{ github.workflow_sha }} + path: trusted-release-policy + persist-credentials: false + sparse-checkout: scripts/ci/release_dependency_evidence.py + sparse-checkout-cone-mode: false + - name: Check dependency graph availability id: dependency_graph env: @@ -133,14 +143,16 @@ jobs: )" if [ "$status" = "200" ]; then + evidence_file="${RUNNER_TEMP}/dependency-graph-compare.json" + install -m 0444 "$response_file" "$evidence_file" + echo "evidence_file=$evidence_file" >>"$GITHUB_OUTPUT" echo "available=true" >>"$GITHUB_OUTPUT" exit 0 fi if [ "$status" = "403" ] || [ "$status" = "404" ]; then - echo "::warning::Dependency graph compare returned HTTP ${status} for ${REPOSITORY}; skipping the dependency-review hard gate (GitHub Dependency Graph, or GitHub Advanced Security on a private repository, is unavailable)." - echo "available=false" >>"$GITHUB_OUTPUT" - exit 0 + echo "::error::Dependency graph compare returned HTTP ${status} for ${REPOSITORY}; release dependency evidence is unavailable, so the gate cannot pass." + exit 1 fi echo "::error::Dependency graph availability check failed with HTTP ${status}. This is not a 'graph unavailable' response (403/404) -- treating it as a genuine failure instead of silently skipping the security gate." @@ -155,9 +167,39 @@ jobs: fail-on-severity: ${{ inputs.fail_on_severity }} allow-ghsas: ${{ inputs.allow_ghsas }} comment-summary-in-pr: ${{ inputs.comment_summary_in_pr }} + deny-licenses: GPL-1.0-only, GPL-1.0-or-later, GPL-2.0-only, GPL-2.0-or-later, GPL-3.0-only, GPL-3.0-or-later, LGPL-2.0-only, LGPL-2.0-or-later, LGPL-2.1-only, LGPL-2.1-or-later, LGPL-3.0-only, LGPL-3.0-or-later, AGPL-1.0-only, AGPL-1.0-or-later, AGPL-3.0-only, AGPL-3.0-or-later - - name: Dependency graph unavailable note - if: steps.dependency_graph.outputs.available != 'true' && github.event_name == 'pull_request' + - name: Bind dependency-by-dependency security evidence to exact head + if: steps.dependency_graph.outputs.available == 'true' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + REPOSITORY: ${{ github.repository }} + EVIDENCE_FILE: ${{ steps.dependency_graph.outputs.evidence_file }} + shell: bash run: | - echo "Dependency Review requires GitHub Dependency Graph to be enabled for this repository (and, on private repositories, GitHub Advanced Security)." - echo "Other required dependency-vulnerability gates (OSV-Scanner, Scorecard) remain the blocking coverage until Dependency Graph is available here." + set -euo pipefail + verifier="trusted-release-policy/scripts/ci/release_dependency_evidence.py" + if [ ! -f "$verifier" ] || [ -L "$verifier" ]; then + echo "::error::Trusted release dependency evidence verifier is missing or unsafe." + exit 2 + fi + if [ -z "$EVIDENCE_FILE" ] || [ ! -f "$EVIDENCE_FILE" ] || [ -L "$EVIDENCE_FILE" ]; then + echo "::error::Dependency-by-dependency evidence is missing or unsafe." + exit 2 + fi + python3 -I "$verifier" \ + --input "$EVIDENCE_FILE" \ + --output release-dependency-evidence.json \ + --repository "$REPOSITORY" \ + --base-sha "$BASE_SHA" \ + --head-sha "$HEAD_SHA" + + - name: Upload exact-head release dependency evidence + if: steps.dependency_graph.outputs.available == 'true' + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: release-dependency-evidence-${{ github.event.pull_request.head.sha }} + path: release-dependency-evidence.json + if-no-files-found: error + retention-days: 30 diff --git a/scripts/ci/release_dependency_evidence.py b/scripts/ci/release_dependency_evidence.py new file mode 100644 index 0000000000..64fa49a9f7 --- /dev/null +++ b/scripts/ci/release_dependency_evidence.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Bind release dependency-review evidence to an exact pull-request revision. + +The GitHub dependency-graph compare response contains one row per changed +dependency, including direct/transitive relationship, manifest, license, and +known vulnerabilities. This verifier rejects incomplete rows and forbidden +GNU-family licenses before emitting a deterministic machine-readable receipt. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any, Sequence + + +FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +FORBIDDEN_LICENSE_RE = re.compile( + r"(?:^|[^A-Z])(?:A?GPL|LGPL)(?:[-+.0-9]|$)", re.IGNORECASE +) +class EvidenceError(ValueError): + """Raised when release dependency evidence is absent or incomplete.""" + + +def _required_text(row: dict[str, Any], key: str, index: int) -> str: + """Return a non-empty string field or fail with its row location.""" + value = row.get(key) + if not isinstance(value, str) or not value.strip(): + raise EvidenceError(f"dependency[{index}].{key} is required") + return value.strip() + + +def dependency_rows(payload: Any) -> list[dict[str, Any]]: + """Extract the compare API's dependency rows without accepting ambiguity.""" + rows = payload.get("dependencies") if isinstance(payload, dict) else payload + if not isinstance(rows, list): + raise EvidenceError("dependency evidence must be a JSON array or dependencies array") + if any(not isinstance(row, dict) for row in rows): + raise EvidenceError("every dependency evidence row must be an object") + return rows + + +def build_receipt( + payload: Any, *, repository: str, base_sha: str, head_sha: str +) -> dict[str, Any]: + """Validate every dependency and build an exact-head evidence receipt.""" + if repository.count("/") != 1: + raise EvidenceError("repository must be owner/name") + if not FULL_SHA_RE.fullmatch(base_sha) or not FULL_SHA_RE.fullmatch(head_sha): + raise EvidenceError("base_sha and head_sha must be full lowercase commit SHAs") + if base_sha == head_sha: + raise EvidenceError("base_sha and head_sha must differ") + + dependencies: list[dict[str, Any]] = [] + for index, row in enumerate(dependency_rows(payload)): + name = _required_text(row, "name", index) + manifest = _required_text(row, "manifest", index) + license_expression = _required_text(row, "license", index) + if FORBIDDEN_LICENSE_RE.search(license_expression): + raise EvidenceError( + f"forbidden release dependency license: {name}: {license_expression}" + ) + vulnerabilities = row.get("vulnerabilities") + if not isinstance(vulnerabilities, list): + raise EvidenceError(f"dependency[{index}].vulnerabilities must be an array") + dependencies.append( + { + "name": name, + "version": str(row.get("version") or ""), + "manifest": manifest, + # GitHub's compare API returns direct and transitive changes but + # does not expose that relationship in its response schema. + "relationship": str(row.get("relationship") or "not_reported_by_compare_api"), + "license": license_expression, + "change_type": str(row.get("change_type") or "unknown"), + "vulnerabilities": vulnerabilities, + } + ) + + dependencies.sort( + key=lambda item: (item["manifest"], item["name"].lower(), item["version"]) + ) + return { + "schema": "cwl-release-dependency-evidence/v1", + "binding": { + "repository": repository, + "base_sha": base_sha, + "head_sha": head_sha, + }, + "policy": { + "coverage": "all direct and transitive changes returned by GitHub dependency review", + "denied_license_families": ["GPL", "LGPL", "AGPL"], + }, + "dependency_count": len(dependencies), + "dependencies": dependencies, + } + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the exact-head evidence verifier CLI.""" + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--repository", required=True) + parser.add_argument("--base-sha", required=True) + parser.add_argument("--head-sha", required=True) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + """Validate compare evidence and atomically publish its structured receipt.""" + args = parse_args(argv) + try: + if not args.input.is_file() or args.input.is_symlink(): + raise EvidenceError("dependency evidence input is missing or unsafe") + payload = json.loads(args.input.read_text(encoding="utf-8")) + receipt = build_receipt( + payload, + repository=args.repository, + base_sha=args.base_sha, + head_sha=args.head_sha, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + temporary = args.output.with_suffix(args.output.suffix + ".tmp") + temporary.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") + temporary.replace(args.output) + except (EvidenceError, OSError, json.JSONDecodeError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 9eed5e2e71..149160b609 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -251,13 +251,17 @@ sanitize_remediation_evidence_claims() { return 2 fi if [ -z "$log_file" ] || [ ! -f "$log_file" ] || [ -L "$log_file" ]; then - return 0 + echo "ERROR: Strix evidence log is missing or unsafe: $log_file" >&2 + return 2 fi if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then - return 0 + echo "ERROR: Strix evidence report root is missing or unsafe: $report_root" >&2 + return 2 fi + local report_count=0 while IFS= read -r -d '' report_file; do + report_count=$((report_count + 1)) python3 -I "$binder" sanitize-report \ --report-file "$report_file" \ --log-file "$log_file" \ @@ -268,6 +272,10 @@ sanitize_remediation_evidence_claims() { done < <( find "$report_root" \( -type f -name 'penetration_test_report.md' -o -type f -name 'vulnerabilities.json' -o -path '*/vulnerabilities/*.md' \) -print0 ) + if [ "$report_count" -eq 0 ]; then + echo "ERROR: Strix structured evidence report is missing under $report_root" >&2 + return 2 + fi } has_strix_report_failure_signal() { diff --git a/tests/test_dependency_review_reusable_workflow_contract.py b/tests/test_dependency_review_reusable_workflow_contract.py index cadefcb8ac..a32e79b6e1 100644 --- a/tests/test_dependency_review_reusable_workflow_contract.py +++ b/tests/test_dependency_review_reusable_workflow_contract.py @@ -43,28 +43,29 @@ def test_declares_workflow_call_with_four_inputs_and_recorded_defaults() -> None def test_step_order_is_harden_then_checkout_then_preflight_then_gated_steps() -> None: - """harden-runner -> checkout -> dependency-graph preflight -> conditional gate/note.""" + """Trusted setup precedes preflight, review, evidence binding, and upload.""" workflow = _workflow_text() order = [ "Harden the runner", "actions/checkout@", "Check dependency graph availability", "Dependency review", - "Dependency graph unavailable note", + "Bind dependency-by-dependency security evidence", + "Upload exact-head release dependency evidence", ] positions = [workflow.index(marker) for marker in order] assert positions == sorted(positions), "steps are out of order" -def test_dependency_review_and_note_steps_are_mutually_exclusive_on_availability() -> None: - """The gate and the fallback note must never both run.""" +def test_dependency_review_only_runs_after_successful_evidence_preflight() -> None: + """The review cannot run without a successful dependency evidence response.""" workflow = _workflow_text() assert ( "if: steps.dependency_graph.outputs.available == 'true'\n" " continue-on-error: ${{ inputs.continue_on_error }}" in workflow ) - assert "if: steps.dependency_graph.outputs.available != 'true'" in workflow + assert "release dependency evidence is unavailable, so the gate cannot pass" in workflow def test_inputs_are_forwarded_to_the_dependency_review_action() -> None: @@ -75,6 +76,25 @@ def test_inputs_are_forwarded_to_the_dependency_review_action() -> None: assert "comment-summary-in-pr: ${{ inputs.comment_summary_in_pr }}" in workflow +def test_forbidden_gnu_family_licenses_are_denied_by_the_blocking_action() -> None: + """Direct and transitive dependency changes must reject GPL-family licenses.""" + workflow = _workflow_text() + deny_line = next(line for line in workflow.splitlines() if "deny-licenses:" in line) + for family in ("GPL-3.0-only", "LGPL-3.0-only", "AGPL-3.0-only"): + assert family in deny_line + + +def test_exact_head_structured_evidence_is_mandatory_and_uploaded() -> None: + """A passing review must publish dependency rows bound to exact base/head SHAs.""" + workflow = _workflow_text() + assert "ref: ${{ github.workflow_sha }}" in workflow + assert "scripts/ci/release_dependency_evidence.py" in workflow + assert "--base-sha \"$BASE_SHA\"" in workflow + assert "--head-sha \"$HEAD_SHA\"" in workflow + assert "if-no-files-found: error" in workflow + assert "Dependency-by-dependency evidence is missing or unsafe" in workflow + + def test_harden_runner_audits_egress() -> None: """naruon's harden-runner step applies uniformly, not only to that one caller.""" workflow = _workflow_text() @@ -112,11 +132,10 @@ def test_availability_check_uses_the_dependency_graph_compare_api() -> None: def test_availability_check_distinguishes_unavailable_from_genuine_failure() -> None: - """403/404 means 'unavailable, skip gracefully'; any other status must hard-fail - the job instead of silently treating a real error the same as unavailability.""" + """Unavailable and unexpected API responses both fail the release gate.""" workflow = _workflow_text() assert 'if [ "$status" = "403" ] || [ "$status" = "404" ]' in workflow - assert "available=false" in workflow + assert "::error::Dependency graph compare returned HTTP" in workflow assert "::error::Dependency graph availability check failed with HTTP" in workflow assert "exit 1" in workflow diff --git a/tests/test_release_dependency_evidence.py b/tests/test_release_dependency_evidence.py new file mode 100644 index 0000000000..92414e469f --- /dev/null +++ b/tests/test_release_dependency_evidence.py @@ -0,0 +1,124 @@ +"""Tests for exact-head release dependency security evidence.""" + +from __future__ import annotations + +import json + +import pytest + +from scripts.ci import release_dependency_evidence as evidence + + +BASE = "a" * 40 +HEAD = "b" * 40 + + +def _row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "name": "anyio", + "version": "4.14.2", + "manifest": "requirements.txt", + "relationship": "transitive", + "license": "MIT", + "change_type": "updated", + "vulnerabilities": [], + } + row.update(overrides) + return row + + +def test_receipt_binds_each_dependency_change_to_exact_head() -> None: + receipt = evidence.build_receipt( + [_row(), _row(name="fastapi", relationship="direct")], + repository="ContextualWisdomLab/fast-mlsirm", + base_sha=BASE, + head_sha=HEAD, + ) + assert receipt["binding"]["base_sha"] == BASE + assert receipt["binding"]["head_sha"] == HEAD + assert {row["relationship"] for row in receipt["dependencies"]} == { + "direct", + "transitive", + } + + +def test_compare_api_row_without_relationship_remains_structured() -> None: + """The documented compare response omits relationship but still covers the row.""" + row = _row() + row.pop("relationship") + receipt = evidence.build_receipt( + [row], + repository="ContextualWisdomLab/fast-mlsirm", + base_sha=BASE, + head_sha=HEAD, + ) + assert receipt["dependencies"][0]["relationship"] == "not_reported_by_compare_api" + + +@pytest.mark.parametrize( + "license_expression", + ["GPL-3.0-only", "LGPL-2.1-or-later", "AGPL-3.0-only"], +) +def test_forbidden_gnu_family_license_fails_closed(license_expression: str) -> None: + with pytest.raises(evidence.EvidenceError, match="forbidden release dependency"): + evidence.build_receipt( + [_row(license=license_expression)], + repository="ContextualWisdomLab/fast-mlsirm", + base_sha=BASE, + head_sha=HEAD, + ) + + +@pytest.mark.parametrize("missing", ["manifest", "license"]) +def test_missing_dependency_evidence_field_fails_closed(missing: str) -> None: + row = _row() + row.pop(missing) + with pytest.raises(evidence.EvidenceError, match=missing): + evidence.build_receipt( + [row], + repository="ContextualWisdomLab/fast-mlsirm", + base_sha=BASE, + head_sha=HEAD, + ) + + +def test_cli_rejects_missing_evidence_file(tmp_path, capsys) -> None: + output = tmp_path / "receipt.json" + result = evidence.main( + [ + "--input", + str(tmp_path / "missing.json"), + "--output", + str(output), + "--repository", + "ContextualWisdomLab/fast-mlsirm", + "--base-sha", + BASE, + "--head-sha", + HEAD, + ] + ) + assert result == 2 + assert not output.exists() + assert "missing or unsafe" in capsys.readouterr().err + + +def test_cli_writes_structured_receipt(tmp_path) -> None: + source = tmp_path / "dependencies.json" + output = tmp_path / "receipt.json" + source.write_text(json.dumps([_row()]), encoding="utf-8") + assert evidence.main( + [ + "--input", + str(source), + "--output", + str(output), + "--repository", + "ContextualWisdomLab/fast-mlsirm", + "--base-sha", + BASE, + "--head-sha", + HEAD, + ] + ) == 0 + assert json.loads(output.read_text())["schema"] == "cwl-release-dependency-evidence/v1" diff --git a/tests/test_strix_mandatory_evidence_contract.py b/tests/test_strix_mandatory_evidence_contract.py new file mode 100644 index 0000000000..0882b59b5a --- /dev/null +++ b/tests/test_strix_mandatory_evidence_contract.py @@ -0,0 +1,19 @@ +"""Fail-closed contract for mandatory Strix evidence inputs.""" + +from pathlib import Path + + +GATE = Path("scripts/ci/strix_quick_gate.sh") + + +def test_missing_log_report_root_and_structured_report_are_hard_errors() -> None: + """Absent evidence must return configuration failure, never success/neutral.""" + source = GATE.read_text(encoding="utf-8") + assert "Strix evidence log is missing or unsafe" in source + assert "Strix evidence report root is missing or unsafe" in source + assert "Strix structured evidence report is missing" in source + start = source.index("sanitize_remediation_evidence_claims()") + end = source.index("\nhas_strix_report_failure_signal()", start) + function = source[start:end] + assert function.count("return 2") >= 4 + assert "return 0" not in function From 1a86be0a24cb397439fffe23936c93d3b68abb76 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 23 Sep 2026 03:07:08 +0900 Subject: [PATCH 4/7] fix(security): close release evidence review gaps --- .../agent-review-runtime-quality-ci.yml | 1 + scripts/ci/release_dependency_evidence.py | 28 ++++++++++++- scripts/ci/strix_quick_gate.sh | 13 ++++++- tests/test_release_dependency_evidence.py | 36 +++++++++++++++++ .../test_strix_mandatory_evidence_contract.py | 39 +++++++++++++++++++ 5 files changed, 115 insertions(+), 2 deletions(-) diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 82e1d60fa8..3cbcf3a6b0 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -36,6 +36,7 @@ on: - "tests/test_docs_only_pr_runner_admission.py" - "tests/test_strix_changed_path_policy.py" - "tests/test_strix_evidence_binding.py" + - "tests/test_strix_mandatory_evidence_contract.py" - "tests/test_strix_model_behavior_error.py" - "tests/test_strix_nvidia_nim_not_found_fallback.py" - "tests/test_strix_workflow_dependency_hashes.py" diff --git a/scripts/ci/release_dependency_evidence.py b/scripts/ci/release_dependency_evidence.py index 64fa49a9f7..e644fcfe06 100644 --- a/scripts/ci/release_dependency_evidence.py +++ b/scripts/ci/release_dependency_evidence.py @@ -16,11 +16,16 @@ from pathlib import Path from typing import Any, Sequence +from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression + FULL_SHA_RE = re.compile(r"^[0-9a-f]{40}$") FORBIDDEN_LICENSE_RE = re.compile( r"(?:^|[^A-Z])(?:A?GPL|LGPL)(?:[-+.0-9]|$)", re.IGNORECASE ) +UNVERIFIABLE_LICENSE_RE = re.compile(r"(?:^|[^A-Za-z])LicenseRef-", re.IGNORECASE) + + class EvidenceError(ValueError): """Raised when release dependency evidence is absent or incomplete.""" @@ -43,6 +48,25 @@ def dependency_rows(payload: Any) -> list[dict[str, Any]]: return rows +def validate_license_expression(value: str, *, dependency_name: str) -> str: + """Return canonical SPDX or fail closed on unknown/custom license evidence.""" + if UNVERIFIABLE_LICENSE_RE.search(value): + raise EvidenceError( + f"unverifiable release dependency license: {dependency_name}: {value}" + ) + try: + canonical = canonicalize_license_expression(value) + except InvalidLicenseExpression as error: + raise EvidenceError( + f"invalid or unknown release dependency license: {dependency_name}: {value}" + ) from error + if UNVERIFIABLE_LICENSE_RE.search(canonical): + raise EvidenceError( + f"unverifiable release dependency license: {dependency_name}: {value}" + ) + return canonical + + def build_receipt( payload: Any, *, repository: str, base_sha: str, head_sha: str ) -> dict[str, Any]: @@ -58,7 +82,9 @@ def build_receipt( for index, row in enumerate(dependency_rows(payload)): name = _required_text(row, "name", index) manifest = _required_text(row, "manifest", index) - license_expression = _required_text(row, "license", index) + license_expression = validate_license_expression( + _required_text(row, "license", index), dependency_name=name + ) if FORBIDDEN_LICENSE_RE.search(license_expression): raise EvidenceError( f"forbidden release dependency license: {name}: {license_expression}" diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 149160b609..14f671885d 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -238,6 +238,15 @@ PY done } +reject_tree_symlinks() { + local root="$1" + local label="$2" + if find "$root" -type l -print -quit | grep -q .; then + echo "ERROR: $label contains a symbolic link: $root" >&2 + return 2 + fi +} + # Issue #2168: reject "already applied" remediation prose when apply_patch # missed the materialized scan workspace. Uses scripts/ci/strix_evidence_binding.py. sanitize_remediation_evidence_claims() { @@ -258,6 +267,7 @@ sanitize_remediation_evidence_claims() { echo "ERROR: Strix evidence report root is missing or unsafe: $report_root" >&2 return 2 fi + reject_tree_symlinks "$report_root" "Strix evidence report root" || return 2 local report_count=0 while IFS= read -r -d '' report_file; do @@ -270,7 +280,7 @@ sanitize_remediation_evidence_claims() { return 2 } done < <( - find "$report_root" \( -type f -name 'penetration_test_report.md' -o -type f -name 'vulnerabilities.json' -o -path '*/vulnerabilities/*.md' \) -print0 + find "$report_root" \( -type f -name 'penetration_test_report.md' -o -type f -name 'vulnerabilities.json' -o -type f -path '*/vulnerabilities/*.md' \) -print0 ) if [ "$report_count" -eq 0 ]; then echo "ERROR: Strix structured evidence report is missing under $report_root" >&2 @@ -2960,6 +2970,7 @@ PY rc=$? set -e if [ -d "$STRIX_SCAN_OUTPUT_DIR" ] && [ ! -L "$STRIX_SCAN_OUTPUT_DIR" ]; then + reject_tree_symlinks "$STRIX_SCAN_OUTPUT_DIR" "Strix scan output" || return 2 cp -R -- "$STRIX_SCAN_OUTPUT_DIR"/. "$ACTIVE_REPORTS_DIR"/ fi local end_epoch diff --git a/tests/test_release_dependency_evidence.py b/tests/test_release_dependency_evidence.py index 92414e469f..59fb2bd8f1 100644 --- a/tests/test_release_dependency_evidence.py +++ b/tests/test_release_dependency_evidence.py @@ -69,6 +69,42 @@ def test_forbidden_gnu_family_license_fails_closed(license_expression: str) -> N ) +@pytest.mark.parametrize( + "license_expression", + ["OTHER", "NOASSERTION", "LicenseRef-Proprietary", "MIT OR made-up-license"], +) +def test_unknown_or_unverifiable_license_fails_closed( + license_expression: str, +) -> None: + with pytest.raises(evidence.EvidenceError, match="invalid|unknown|unverifiable"): + evidence.build_receipt( + [_row(license=license_expression)], + repository="ContextualWisdomLab/fast-mlsirm", + base_sha=BASE, + head_sha=HEAD, + ) + + +@pytest.mark.parametrize( + ("license_expression", "canonical"), + [ + ("MIT", "MIT"), + ("MIT OR Apache-2.0", "MIT OR Apache-2.0"), + ("(MIT AND BSD-3-Clause)", "(MIT AND BSD-3-Clause)"), + ], +) +def test_valid_spdx_expressions_are_canonicalized( + license_expression: str, canonical: str +) -> None: + receipt = evidence.build_receipt( + [_row(license=license_expression)], + repository="ContextualWisdomLab/fast-mlsirm", + base_sha=BASE, + head_sha=HEAD, + ) + assert receipt["dependencies"][0]["license"] == canonical + + @pytest.mark.parametrize("missing", ["manifest", "license"]) def test_missing_dependency_evidence_field_fails_closed(missing: str) -> None: row = _row() diff --git a/tests/test_strix_mandatory_evidence_contract.py b/tests/test_strix_mandatory_evidence_contract.py index 0882b59b5a..781c8c2b08 100644 --- a/tests/test_strix_mandatory_evidence_contract.py +++ b/tests/test_strix_mandatory_evidence_contract.py @@ -1,5 +1,6 @@ """Fail-closed contract for mandatory Strix evidence inputs.""" +import subprocess from pathlib import Path @@ -17,3 +18,41 @@ def test_missing_log_report_root_and_structured_report_are_hard_errors() -> None function = source[start:end] assert function.count("return 2") >= 4 assert "return 0" not in function + + +def test_recursive_symlinks_are_rejected_before_copy_or_sanitization() -> None: + """Nested symlinks cannot escape either evidence boundary.""" + source = GATE.read_text(encoding="utf-8") + assert 'reject_tree_symlinks "$report_root"' in source + assert 'reject_tree_symlinks "$STRIX_SCAN_OUTPUT_DIR"' in source + symlink_check = source.index('reject_tree_symlinks "$STRIX_SCAN_OUTPUT_DIR"') + evidence_copy = source.index( + 'cp -R -- "$STRIX_SCAN_OUTPUT_DIR"/. "$ACTIVE_REPORTS_DIR"/', symlink_check + ) + assert symlink_check < evidence_copy + assert "-o -type f -path '*/vulnerabilities/*.md'" in source + + +def test_recursive_symlink_guard_rejects_nested_link_at_runtime(tmp_path) -> None: + """The production guard returns rc=2 for a link below a regular root.""" + root = tmp_path / "reports" + nested = root / "nested" + nested.mkdir(parents=True) + (nested / "escape").symlink_to(tmp_path / "outside") + + source = GATE.read_text(encoding="utf-8") + start = source.index("reject_tree_symlinks()") + end = source.index("\n# Issue #2168", start) + function = source[start:end] + harness = tmp_path / "guard.sh" + harness.write_text( + "set -u\n" + + function + + f'\nreject_tree_symlinks "{root}" "test evidence"\n', + encoding="utf-8", + ) + result = subprocess.run( + ["bash", str(harness)], text=True, capture_output=True, check=False + ) + assert result.returncode == 2 + assert "contains a symbolic link" in result.stderr From b5b3b2cc95bc058a1a9249a86ddeb21562549694 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 23 Sep 2026 03:32:06 +0900 Subject: [PATCH 5/7] fix(security): keep dependency review fail closed --- .github/workflows/dependency-review.yml | 16 ++-- scripts/ci/release_dependency_evidence.py | 86 ++++++++++++++++++- ...dency_review_reusable_workflow_contract.py | 24 ++++-- tests/test_release_dependency_evidence.py | 60 +++++++++++++ 4 files changed, 169 insertions(+), 17 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index c34fd5a6d4..a29f306c6b 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -61,10 +61,8 @@ on: default: "" continue_on_error: description: >- - Whether the dependency-review step itself is allowed to fail - without failing the job (argos's original behavior, which relies - on a separate blocking OSV-Scanner gate instead of this one). - Default false makes the dependency-review step itself blocking. + Deprecated compatibility input. It is intentionally ignored: a + caller cannot weaken the formal release dependency hard gate. required: false type: boolean default: false @@ -160,8 +158,8 @@ jobs: exit 1 - name: Dependency review + id: dependency_review if: steps.dependency_graph.outputs.available == 'true' - continue-on-error: ${{ inputs.continue_on_error }} uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 with: fail-on-severity: ${{ inputs.fail_on_severity }} @@ -170,12 +168,13 @@ jobs: deny-licenses: GPL-1.0-only, GPL-1.0-or-later, GPL-2.0-only, GPL-2.0-or-later, GPL-3.0-only, GPL-3.0-or-later, LGPL-2.0-only, LGPL-2.0-or-later, LGPL-2.1-only, LGPL-2.1-or-later, LGPL-3.0-only, LGPL-3.0-or-later, AGPL-1.0-only, AGPL-1.0-or-later, AGPL-3.0-only, AGPL-3.0-or-later - name: Bind dependency-by-dependency security evidence to exact head - if: steps.dependency_graph.outputs.available == 'true' + if: always() && steps.dependency_graph.outputs.available == 'true' env: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} REPOSITORY: ${{ github.repository }} EVIDENCE_FILE: ${{ steps.dependency_graph.outputs.evidence_file }} + DEPENDENCY_REVIEW_OUTCOME: ${{ steps.dependency_review.outcome }} shell: bash run: | set -euo pipefail @@ -193,10 +192,11 @@ jobs: --output release-dependency-evidence.json \ --repository "$REPOSITORY" \ --base-sha "$BASE_SHA" \ - --head-sha "$HEAD_SHA" + --head-sha "$HEAD_SHA" \ + --dependency-review-outcome "$DEPENDENCY_REVIEW_OUTCOME" - name: Upload exact-head release dependency evidence - if: steps.dependency_graph.outputs.available == 'true' + if: always() && steps.dependency_graph.outputs.available == 'true' uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: release-dependency-evidence-${{ github.event.pull_request.head.sha }} diff --git a/scripts/ci/release_dependency_evidence.py b/scripts/ci/release_dependency_evidence.py index e644fcfe06..f222085c7b 100644 --- a/scripts/ci/release_dependency_evidence.py +++ b/scripts/ci/release_dependency_evidence.py @@ -125,6 +125,59 @@ def build_receipt( } +def build_rejection_receipt( + payload: Any, + *, + repository: str, + base_sha: str, + head_sha: str, + reason: str, +) -> dict[str, Any]: + """Preserve deterministic per-dependency evidence for a rejected review.""" + rows = dependency_rows(payload) + dependencies = [ + { + "name": str(row.get("name") or ""), + "version": str(row.get("version") or ""), + "manifest": str(row.get("manifest") or ""), + "relationship": str( + row.get("relationship") or "not_reported_by_compare_api" + ), + "license": str(row.get("license") or ""), + "change_type": str(row.get("change_type") or "unknown"), + "vulnerabilities": ( + row.get("vulnerabilities") + if isinstance(row.get("vulnerabilities"), list) + else [] + ), + } + for row in rows + ] + dependencies.sort( + key=lambda item: (item["manifest"], item["name"].lower(), item["version"]) + ) + return { + "schema": "cwl-release-dependency-evidence/v1", + "binding": { + "repository": repository, + "base_sha": base_sha, + "head_sha": head_sha, + }, + "result": "rejected", + "rejection_reason": reason, + "dependency_count": len(dependencies), + "dependencies": dependencies, + } + + +def write_receipt(output: Path, receipt: dict[str, Any]) -> None: + """Atomically publish one accepted or rejected exact-head receipt.""" + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_suffix(output.suffix + ".tmp") + temporary.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") + temporary.replace(output) + + def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: """Parse the exact-head evidence verifier CLI.""" parser = argparse.ArgumentParser() @@ -133,12 +186,18 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--repository", required=True) parser.add_argument("--base-sha", required=True) parser.add_argument("--head-sha", required=True) + parser.add_argument( + "--dependency-review-outcome", + choices=("success", "failure", "cancelled", "skipped"), + default="success", + ) return parser.parse_args(argv) def main(argv: Sequence[str] | None = None) -> int: """Validate compare evidence and atomically publish its structured receipt.""" args = parse_args(argv) + payload: Any = None try: if not args.input.is_file() or args.input.is_symlink(): raise EvidenceError("dependency evidence input is missing or unsafe") @@ -149,12 +208,31 @@ def main(argv: Sequence[str] | None = None) -> int: base_sha=args.base_sha, head_sha=args.head_sha, ) - args.output.parent.mkdir(parents=True, exist_ok=True) - temporary = args.output.with_suffix(args.output.suffix + ".tmp") - temporary.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") - temporary.replace(args.output) + if args.dependency_review_outcome != "success": + receipt["result"] = "rejected" + receipt["rejection_reason"] = ( + "dependency-review action outcome: " + args.dependency_review_outcome + ) + write_receipt(args.output, receipt) + return 2 + receipt["result"] = "accepted" + write_receipt(args.output, receipt) except (EvidenceError, OSError, json.JSONDecodeError) as error: print(f"ERROR: {error}", file=sys.stderr) + if payload is not None: + try: + write_receipt( + args.output, + build_rejection_receipt( + payload, + repository=args.repository, + base_sha=args.base_sha, + head_sha=args.head_sha, + reason=str(error), + ), + ) + except (EvidenceError, OSError): + pass return 2 return 0 diff --git a/tests/test_dependency_review_reusable_workflow_contract.py b/tests/test_dependency_review_reusable_workflow_contract.py index a32e79b6e1..682a0a648f 100644 --- a/tests/test_dependency_review_reusable_workflow_contract.py +++ b/tests/test_dependency_review_reusable_workflow_contract.py @@ -60,14 +60,28 @@ def test_step_order_is_harden_then_checkout_then_preflight_then_gated_steps() -> def test_dependency_review_only_runs_after_successful_evidence_preflight() -> None: """The review cannot run without a successful dependency evidence response.""" workflow = _workflow_text() - assert ( - "if: steps.dependency_graph.outputs.available == 'true'\n" - " continue-on-error: ${{ inputs.continue_on_error }}" - in workflow - ) + assert "if: steps.dependency_graph.outputs.available == 'true'" in workflow assert "release dependency evidence is unavailable, so the gate cannot pass" in workflow +def test_caller_cannot_override_the_dependency_review_hard_gate() -> None: + """The legacy compatibility input never reaches continue-on-error.""" + workflow = _workflow_text() + assert "Deprecated compatibility input" in workflow + assert "continue-on-error: ${{ inputs.continue_on_error }}" not in workflow + + +def test_failure_path_still_binds_and_uploads_rejection_evidence() -> None: + """A rejected dependency-review action cannot suppress its evidence receipt.""" + workflow = _workflow_text() + assert workflow.count( + "if: always() && steps.dependency_graph.outputs.available == 'true'" + ) == 2 + assert "DEPENDENCY_REVIEW_OUTCOME: ${{ steps.dependency_review.outcome }}" in workflow + assert '--dependency-review-outcome "$DEPENDENCY_REVIEW_OUTCOME"' in workflow + assert "if-no-files-found: error" in workflow + + def test_inputs_are_forwarded_to_the_dependency_review_action() -> None: """fail_on_severity, allow_ghsas, and comment_summary_in_pr must reach the action untouched.""" workflow = _workflow_text() diff --git a/tests/test_release_dependency_evidence.py b/tests/test_release_dependency_evidence.py index 59fb2bd8f1..214a7e5b6d 100644 --- a/tests/test_release_dependency_evidence.py +++ b/tests/test_release_dependency_evidence.py @@ -158,3 +158,63 @@ def test_cli_writes_structured_receipt(tmp_path) -> None: ] ) == 0 assert json.loads(output.read_text())["schema"] == "cwl-release-dependency-evidence/v1" + + +def test_cli_rejection_writes_exact_head_per_dependency_evidence(tmp_path) -> None: + source = tmp_path / "dependencies.json" + output = tmp_path / "receipt.json" + source.write_text( + json.dumps([_row(license="NOASSERTION"), _row(name="gpl", license="GPL-3.0-only")]), + encoding="utf-8", + ) + result = evidence.main( + [ + "--input", + str(source), + "--output", + str(output), + "--repository", + "ContextualWisdomLab/fast-mlsirm", + "--base-sha", + BASE, + "--head-sha", + HEAD, + ] + ) + assert result == 2 + receipt = json.loads(output.read_text(encoding="utf-8")) + assert receipt["result"] == "rejected" + assert receipt["binding"] == { + "repository": "ContextualWisdomLab/fast-mlsirm", + "base_sha": BASE, + "head_sha": HEAD, + } + assert receipt["dependency_count"] == 2 + assert {row["name"] for row in receipt["dependencies"]} == {"anyio", "gpl"} + + +def test_dependency_review_failure_writes_rejected_receipt(tmp_path) -> None: + source = tmp_path / "dependencies.json" + output = tmp_path / "receipt.json" + source.write_text(json.dumps([_row()]), encoding="utf-8") + result = evidence.main( + [ + "--input", + str(source), + "--output", + str(output), + "--repository", + "ContextualWisdomLab/fast-mlsirm", + "--base-sha", + BASE, + "--head-sha", + HEAD, + "--dependency-review-outcome", + "failure", + ] + ) + assert result == 2 + receipt = json.loads(output.read_text(encoding="utf-8")) + assert receipt["result"] == "rejected" + assert receipt["dependency_count"] == 1 + assert receipt["rejection_reason"] == "dependency-review action outcome: failure" From 28afd9e0910e35a60ade0cef4fc5389e21266c81 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 23 Sep 2026 06:46:48 +0900 Subject: [PATCH 6/7] test: restore Strix fixture evidence contracts --- scripts/ci/test_strix_quick_gate.sh | 122 +++++++++------------- tests/test_release_dependency_evidence.py | 2 +- 2 files changed, 48 insertions(+), 76 deletions(-) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7a93cffbc7..bfac80d7de 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -13,6 +13,14 @@ REPO_ROOT="$( )" GATE_SCRIPT="$REPO_ROOT/scripts/ci/strix_quick_gate.sh" +install_gate_fixture() { + local destination="$1" + cp "$GATE_SCRIPT" "$destination/strix_quick_gate.sh" + cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$destination/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$destination/strix_evidence_binding.py" + chmod +x "$destination/strix_quick_gate.sh" +} + FAILURES=0 TIMEOUT_TEST_PROCESS_SECONDS="${STRIX_TEST_PROCESS_TIMEOUT_SECONDS:-30}" TIMEOUT_TEST_FAKE_SLEEP_SECONDS="${STRIX_TEST_FAKE_SLEEP_SECONDS:-60}" @@ -3296,9 +3304,7 @@ run_gate_case() { mkdir -p "$bin_dir" "$untrusted_bin_dir" "$repo_root_dir/src" mkdir -p "$repo_root_dir/scripts/ci" local gate_under_test="$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$GATE_SCRIPT" "$gate_under_test" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$gate_under_test" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local path_hijack_log="$tmp_dir/path-hijack.log" cat >"$untrusted_bin_dir/strix" <<'EOF' @@ -3366,6 +3372,20 @@ printf '%s\n' "$target_path" >> "${FAKE_STRIX_TARGET_LOG:?}" STRIX_REPORTS_DIR="${STRIX_REPORTS_DIR:-strix_runs}" +emit_success_report() { + local status="$?" + if [ "$status" -eq 0 ] && [ "${FAKE_STRIX_EMIT_STRUCTURED_REPORT:-1}" = "1" ]; then + mkdir -p "$STRIX_REPORTS_DIR/fake-success" + cat >"$STRIX_REPORTS_DIR/fake-success/penetration_test_report.md" <<'REPORT' +# Penetration Test Report + +Vulnerabilities: 0 +REPORT + fi + exit "$status" +} +trap emit_success_report EXIT + case "${FAKE_STRIX_SCENARIO:?}" in success|runtime-env-forwarding|custom-openai-compatible-preserves-effort|vertex-primary-success-timing-message|direct-openai-gpt-does-not-require-github-models-api-base|pr-executable-integrity-mismatch|pr-executable-group-writable) echo "scan ok" @@ -7026,9 +7046,7 @@ run_pull_request_target_head_scope_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -7174,9 +7192,7 @@ run_pull_request_target_plaintext_runner_token_fails_closed_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -7296,9 +7312,7 @@ run_pull_request_target_bounded_head_context_scope_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -7401,9 +7415,7 @@ run_pull_request_target_changed_context_scope_uses_pr_head_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -7580,9 +7592,7 @@ run_pull_request_target_changed_backend_context_scope_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -7839,9 +7849,7 @@ run_pull_request_target_frontend_email_context_scope_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -8029,9 +8037,7 @@ run_pull_request_target_shallow_head_merge_base_fallback_case() { local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$origin_repo_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -8144,9 +8150,7 @@ run_pull_request_target_aborts_on_pr_head_blob_failure_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local real_git real_git="$(command -v git)" @@ -8268,9 +8272,7 @@ run_pull_request_target_rejects_invalid_sha_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local call_log="$tmp_dir/calls.log" @@ -8361,9 +8363,7 @@ run_pull_request_target_irregular_head_entry_fails_closed_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local call_log="$tmp_dir/calls.log" @@ -8444,9 +8444,7 @@ run_pull_request_target_gitlink_is_explicitly_skipped_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local call_log="$tmp_dir/calls.log" @@ -8526,9 +8524,7 @@ run_full_head_scope_skips_gitlink_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -8640,9 +8636,7 @@ run_pull_request_target_rejects_unsafe_changed_path_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/repo" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local call_log="$tmp_dir/calls.log" @@ -8732,9 +8726,7 @@ run_timeout_cleanup_case() { local workspace_dir="$tmp_dir/workspace" local repo_root_dir="$workspace_dir/smart-crawling-server" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local child_pid_file="$tmp_dir/child.pid" local output_log="$tmp_dir/output.log" @@ -8814,9 +8806,7 @@ run_vertex_model_ignores_untrusted_llm_api_base_file_case() { local llm_api_base_file="$outside_dir/llm_api_base.txt" mkdir -p "$repo_root_dir/scripts/ci" "$allowed_input_dir" "$outside_dir" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -8866,9 +8856,7 @@ run_total_timeout_case() { local workspace_dir="$tmp_dir/workspace" local repo_root_dir="$workspace_dir/smart-crawling-server" mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" local call_count_file="$tmp_dir/calls.log" @@ -9193,9 +9181,7 @@ run_llm_api_base_file_outside_input_root_fails_closed_case() { local llm_api_base_file="$outside_dir/llm_api_base.txt" mkdir -p "$repo_root_dir/scripts/ci" "$allowed_input_dir" "$outside_dir" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -9248,9 +9234,7 @@ run_pr_scoped_llm_api_base_file_config_failure_exits_2_case() { local llm_api_base_file="$outside_dir/llm_api_base.txt" mkdir -p "$repo_root_dir/scripts/ci" "$repo_root_dir/src" "$allowed_input_dir" "$outside_dir" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" printf '%s\n' 'print("one")' >"$repo_root_dir/src/one.py" printf '%s\n' 'print("two")' >"$repo_root_dir/src/two.py" @@ -9309,9 +9293,7 @@ run_required_input_file_outside_input_root_fails_closed_case() { local outside_file="$outside_dir/${file_env}.txt" mkdir -p "$repo_root_dir/scripts/ci" "$allowed_input_dir" "$outside_dir" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -9379,9 +9361,7 @@ run_input_file_root_override_takes_precedence_over_runner_temp_case() { local llm_api_base_file="$explicit_input_root/llm_api_base.txt" mkdir -p "$repo_root_dir/scripts/ci" "$explicit_input_root" "$inherited_runner_temp" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -9433,9 +9413,7 @@ run_stale_report_case() { local llm_api_base_file="$tmp_dir/llm_api_base.txt" mkdir -p "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" mkdir -p "$stale_report_dir" cat >"$stale_report_dir/vuln-0001.md" <<'EOF' @@ -9488,9 +9466,7 @@ run_symlink_report_case() { local llm_api_base_file="$tmp_dir/llm_api_base.txt" mkdir -p "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" mkdir -p "$external_report_dir" "$repo_root_dir/strix_runs" cat >"$external_report_dir/vuln-0001.md" <<'EOF' @@ -9544,9 +9520,7 @@ run_unsafe_target_path_case() { local llm_api_base_file="$tmp_dir/llm_api_base.txt" mkdir -p "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash @@ -9592,9 +9566,7 @@ run_absolute_outside_target_path_case() { local bin_dir="$tmp_dir/bin" local repo_root_dir="$tmp_dir/workspace/smart-crawling-server" mkdir -p "$bin_dir" "$repo_root_dir/src" "$repo_root_dir/scripts/ci" - cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" - chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" + install_gate_fixture "$repo_root_dir/scripts/ci" local fake_strix="$bin_dir/strix" local call_log="$tmp_dir/calls.log" local output_log="$tmp_dir/output.log" diff --git a/tests/test_release_dependency_evidence.py b/tests/test_release_dependency_evidence.py index 214a7e5b6d..c6acc8330e 100644 --- a/tests/test_release_dependency_evidence.py +++ b/tests/test_release_dependency_evidence.py @@ -29,7 +29,7 @@ def _row(**overrides: object) -> dict[str, object]: def test_receipt_binds_each_dependency_change_to_exact_head() -> None: receipt = evidence.build_receipt( - [_row(), _row(name="fastapi", relationship="direct")], + [_row(), _row(name="starlette", relationship="direct")], repository="ContextualWisdomLab/fast-mlsirm", base_sha=BASE, head_sha=HEAD, From ca517157e974b1fb99fb29ac5731276743601032 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 23 Sep 2026 07:46:26 +0900 Subject: [PATCH 7/7] test: emit evidence for PR head scope fixtures --- scripts/ci/test_strix_quick_gate.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index bfac80d7de..da742095b3 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -7106,6 +7106,12 @@ else exit 68 fi fi +mkdir -p "${STRIX_REPORTS_DIR:?}/fake-success" +cat >"${STRIX_REPORTS_DIR:?}/fake-success/penetration_test_report.md" <<'REPORT' +# Penetration Test Report + +Vulnerabilities: 0 +REPORT echo "scan ok with PR head content" EOF chmod +x "$fake_strix"