diff --git a/CHANGELOG.d/20260930-strix-unverified-dependency.md b/CHANGELOG.d/20260930-strix-unverified-dependency.md new file mode 100644 index 0000000000..44112822d4 --- /dev/null +++ b/CHANGELOG.d/20260930-strix-unverified-dependency.md @@ -0,0 +1,9 @@ +## Changed + +- The Strix gate no longer fails closed on a finding that names only packages + the repository does not depend on. When such a finding has no file location, + every package in its Target, Package and Introduced By fields is absent from + every dependency manifest and lockfile, and the pull request changes no + manifest, the gate records it as unverified with a warning annotation. A + model reported a lodash CVE (itself misattributed) on a Rust/Python + repository with no JavaScript dependencies. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index ccf08f48eb..6586482d8c 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2260,6 +2260,23 @@ vulnerability_file_is_below_threshold() { [ "$report_rank" -ge 0 ] && [ "$report_rank" -lt "$threshold_rank" ] } +# A model can name a CVE in a package the repository never depends on +# (fast-mlsirm#2246: "CVE-2024-1234 in lodash 4.17.20" on a Rust/Python tree). +# Such a finding has no file location; record it as unverified instead of +# failing closed, but only when every package it names is absent from every +# dependency manifest and lockfile, and only when the pull request changes no +# dependency manifest (REPO_ROOT is the base checkout, so a dependency the PR +# adds would otherwise look absent). +vulnerability_file_is_unverified_dependency() { + local changed_file + for changed_file in "${CHANGED_FILES[@]}"; do + if is_dependency_manifest_path "$changed_file"; then + return 1 + fi + done + python3 -I "$SCRIPT_DIR/strix_unverified_dependency.py" "$1" "$REPO_ROOT" +} + evaluate_pull_request_findings() { PR_FINDINGS_DECISION="not_applicable" if ! is_pull_request_event; then @@ -2276,6 +2293,7 @@ evaluate_pull_request_findings() { local found_baseline_threshold_finding=0 local found_changed_manifest_only_threshold_finding=0 local found_retryable_model_inconsistency=0 + local found_unverified_dependency=0 local found_any_vuln_file=0 local run_dir vulnerabilities_dir vuln_file line severity rank for run_dir in "$STRIX_REPORTS_DIR"/*; do @@ -2310,6 +2328,10 @@ evaluate_pull_request_findings() { mapfile -t vulnerability_location_records < <(extract_vulnerability_location_records "$vuln_file") mapfile -t vulnerability_locations < <(extract_vulnerability_locations "$vuln_file") if [ "${#vulnerability_locations[@]}" -eq 0 ]; then + if vulnerability_file_is_unverified_dependency "$vuln_file"; then + found_unverified_dependency=1 + continue + fi PR_FINDINGS_DECISION="block_unmapped" echo "Unable to map Strix findings to changed files; failing closed for pull request." >&2 return 1 @@ -2349,6 +2371,12 @@ evaluate_pull_request_findings() { done done + if [ "$found_baseline_threshold_finding" -eq 0 ] && [ "$found_changed_manifest_only_threshold_finding" -eq 0 ] && + [ "$found_unverified_dependency" -eq 1 ] && vulnerability_file_is_unverified_dependency "$STRIX_LOG"; then + PR_FINDINGS_DECISION="allow_unverified_dependency" + return 0 + fi + if [ "$found_baseline_threshold_finding" -eq 0 ] && [ "$found_changed_manifest_only_threshold_finding" -eq 0 ]; then rank="$(extract_max_severity_rank "$STRIX_LOG")" if [ "$rank" -lt 0 ]; then @@ -2443,6 +2471,9 @@ has_unmapped_threshold_report() { local vulnerability_locations=() mapfile -t vulnerability_locations < <(extract_vulnerability_locations "$vuln_file") if [ "${#vulnerability_locations[@]}" -eq 0 ]; then + if vulnerability_file_is_unverified_dependency "$vuln_file"; then + continue + fi return 0 fi done @@ -2996,7 +3027,8 @@ PY return 1 fi if has_blocking_vulnerability_reports; then - if ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then + if ! evaluate_pull_request_findings || + { [ "$PR_FINDINGS_DECISION" != "allow_baseline" ] && [ "$PR_FINDINGS_DECISION" != "allow_unverified_dependency" ]; }; then echo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 return 1 fi @@ -3653,7 +3685,7 @@ has_blocking_vulnerability_reports() { fail_reported_vulnerabilities_before_fallback_success() { case "$PR_FINDINGS_DECISION" in - allow_baseline) + allow_baseline | allow_unverified_dependency) return 1 ;; esac @@ -4586,7 +4618,7 @@ run_current_target_scan() { fi if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && - [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then + { [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] || [ "$PR_FINDINGS_DECISION" = "allow_unverified_dependency" ]; }; then echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 return 1 fi diff --git a/scripts/ci/strix_unverified_dependency.py b/scripts/ci/strix_unverified_dependency.py new file mode 100755 index 0000000000..bc4c781708 --- /dev/null +++ b/scripts/ci/strix_unverified_dependency.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Classify a Strix finding about a package the repository does not depend on. + +A model can report a CVE in a dependency the scanned repository never uses +(fast-mlsirm#2246: "CVE-2024-1234 in lodash 4.17.20" on a Rust/Python tree). +Such a finding has no file location, so the gate would fail closed as +unmapped. This helper lets the gate record it as unverified instead, but only +when every package the report names is absent from every dependency manifest +and lockfile in the repository. Reports that name no package are never +affected. + +Usage: strix_unverified_dependency.py REPORT REPO_ROOT +Exit 0 when the finding is unverified, 1 otherwise. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +MANIFEST_NAMES = frozenset({ + "Cargo.toml", "Cargo.lock", "pyproject.toml", "uv.lock", "poetry.lock", "Pipfile", "Pipfile.lock", + "setup.cfg", "setup.py", "package.json", "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", + "pnpm-lock.yaml", "go.mod", "go.sum", "Gemfile", "Gemfile.lock", "composer.json", "composer.lock", + "pom.xml", "build.gradle", "build.gradle.kts", "packages.lock.json", "renv.lock", "DESCRIPTION", +}) +SKIP_DIRS = frozenset({".git", "node_modules", "target", ".venv", "venv", "__pycache__", "vendor"}) +MAX_MANIFEST_BYTES = 8 * 1024 * 1024 + +# A package is a bare name or an npm scope (``@scope/name``); anything with a +# path separator otherwise is a file, not a package. +_PACKAGE = r"(@[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*|[A-Za-z0-9][A-Za-z0-9_.-]*)" +_VERSION = r"v?\d+(?:\.\d+)+[A-Za-z0-9.+-]*" +# Report files use "**Target:** x"; the console log boxes the same fields as +# "│ Target: x │". +_FIELD = r"^[ \t]*│?[ \t]*(?:\*\*)?{name}:(?:\*\*)?[ \t]*" +_END = r"[ \t]*│?[ \t]*$" +# Only Strix's structured dependency fields name packages; free text such as +# "weak TLS in openssl 1.1.1" never does. +FIELD_RES = ( + re.compile(_FIELD.format(name="Target") + rf"{_PACKAGE}(?:@|[ \t]+){_VERSION}{_END}", re.MULTILINE), + re.compile(_FIELD.format(name="Package") + rf"{_PACKAGE}(?:(?:@|[ \t]+){_VERSION})?{_END}", re.MULTILINE), + re.compile(_FIELD.format(name="Introduced By") + rf"{_PACKAGE}@{_VERSION}{_END}", re.MULTILINE), +) + + +def named_packages(report: str) -> set[str]: + """Return lower-cased package names from the report's dependency fields.""" + return {m.group(1).lower() for regex in FIELD_RES for m in regex.finditer(report)} + + +def _is_requirements(name: str) -> bool: + return name.startswith("requirements") and name.endswith((".txt", ".in")) + + +def _manifest_text(repo_root: Path) -> str: + chunks = [] + for path in repo_root.rglob("*"): + if any(part in SKIP_DIRS for part in path.relative_to(repo_root).parts[:-1]): + continue + if not path.is_file() or path.is_symlink(): + continue + if path.name not in MANIFEST_NAMES and not _is_requirements(path.name): + continue + if path.stat().st_size <= MAX_MANIFEST_BYTES: + chunks.append(path.read_text(encoding="utf-8", errors="replace").lower()) + return "\n".join(chunks) + + +def unverified_dependency_finding(report: str, repo_root: Path) -> bool: + """True when the report names packages and none appears in any manifest. + + Matching is deliberately loose (``node_modules/lodash`` counts), so doubt + keeps the finding. + """ + packages = named_packages(report) + if not packages: + return False + manifests = _manifest_text(repo_root) + return not any(re.search(rf"(? int: + if len(argv) != 3: + print(__doc__, file=sys.stderr) + return 2 + report = Path(argv[1]).read_text(encoding="utf-8", errors="replace") + if unverified_dependency_finding(report, Path(argv[2])): + names = ", ".join(sorted(named_packages(report))) + print(f"::warning::Strix finding names package(s) {names} absent from every dependency manifest and " + "lockfile; recording it as unverified instead of failing closed.", file=sys.stderr) + return 0 + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 80c4832243..c9468ee401 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -23,6 +23,7 @@ materialize_trusted_gate_fixture() { cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$fixture_script_dir/strix_model_utils.sh" cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$fixture_script_dir/strix_evidence_binding.py" cp "$REPO_ROOT/scripts/ci/strix_report_scope.py" "$fixture_script_dir/strix_report_scope.py" + cp "$REPO_ROOT/scripts/ci/strix_unverified_dependency.py" "$fixture_script_dir/strix_unverified_dependency.py" chmod +x "$fixture_script_dir/strix_quick_gate.sh" } TIMEOUT_TEST_PROCESS_SECONDS="${STRIX_TEST_PROCESS_TIMEOUT_SECONDS:-30}" @@ -3307,6 +3308,9 @@ run_gate_case() { local trusted_script_dir="$tmp_dir/trusted-source/scripts/ci" local gate_under_test="$trusted_script_dir/strix_quick_gate.sh" materialize_trusted_gate_fixture "$trusted_script_dir" + if [ "$scenario" = "pr-unverified-dependency-present" ]; then + printf '{"packages": {"node_modules/lodash": {"version": "4.17.20"}}}\n' >"$repo_root_dir/package-lock.json" + fi if [ "$scenario" = "pr-changed-scope-includes-ci-dependency" ]; then # Consumer source under scan; execution still uses the separate trusted runtime. cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" @@ -4955,6 +4959,35 @@ FINDINGS ;; esac ;; + pr-unverified-dependency-lodash | pr-unverified-dependency-present | pr-unverified-dependency-manifest-changed) + # Verbatim vuln-0001.md from the strix-reports artifact of fast-mlsirm#2246 run 36580588738: a free model reported a CVE in a + # package the Rust/Python repository does not depend on. + mkdir -p "$STRIX_REPORTS_DIR/fake-unverified-dependency/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-unverified-dependency/vulnerabilities/vuln-0001.md" <<'EOS' +# CVE-2024-1234 in lodash 4.17.20 (prototype pollution) + +**ID:** vuln-0001 +**Severity:** MEDIUM +**Found:** 2026-09-29 14:22:11 UTC +**Target:** lodash 4.17.20 +**Package:** lodash +**Ecosystem:** npm +**Installed Version:** 4.17.20 +**Fixed Version:** 4.17.21 +**Introduced By:** express@4.18.1 +**Dependency Chain:** express@4.18.1 > lodash@4.17.20 +**CVE:** CVE-2024-1234 +**CWE:** CWE-78 +**CVSS:** 5.6 +**Fix Effort:** Low +EOS + printf '%s\n' \ + '│ Title: CVE-2024-1234 in lodash 4.17.20 (prototype pollution) │' \ + '│ Severity: MEDIUM │' \ + '│ Target: lodash 4.17.20 │' + echo "Penetration test failed: MEDIUM finding in lodash 4.17.20" + exit 1 + ;; pr-baseline-critical-unchanged) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities/vuln-0001.md" <<'EOS' @@ -6166,11 +6199,33 @@ run_github_models_http410_case() { "1" } +run_unverified_dependency_case() { + local scenario="$1" expected_exit expected_message + if [ "$scenario" = "pr-unverified-dependency-lodash" ]; then + expected_exit="0" + expected_message="::warning::Strix finding names package(s) express, lodash absent from every dependency manifest and lockfile; recording it as unverified instead of failing closed." + else + expected_exit="1" + expected_message="Unable to map Strix findings to changed files; failing closed for pull request." + fi + local changed_files="sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + if [ "$scenario" = "pr-unverified-dependency-manifest-changed" ]; then + # A PR that changes a manifest may add the package; keep failing closed. + changed_files="$changed_files"$'\n'"package.json" + fi + run_gate_case "$scenario" "openai/gpt-4o-mini" "" "$expected_exit" "$expected_message" "1" \ + "openai/gpt-4o-mini" "https://example.invalid" "vertex_ai" "__DEFAULT__" "" "0" "MEDIUM" "0" \ + "" "" "1200" "0" "pull_request" "$changed_files" +} + run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in "") return 0 ;; + pr-unverified-dependency-lodash | pr-unverified-dependency-present | pr-unverified-dependency-manifest-changed) + run_unverified_dependency_case "$STRIX_TEST_CASE_FILTER" + ;; success) run_gate_case "success" \ "vertex_ai/ready-primary" \ @@ -12185,6 +12240,10 @@ run_gate_case "pr-critical-changed" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_unverified_dependency_case pr-unverified-dependency-lodash +run_unverified_dependency_case pr-unverified-dependency-present +run_unverified_dependency_case pr-unverified-dependency-manifest-changed + run_gate_case "pr-changed-file-nonintersecting-line" \ "openai/gpt-4o-mini" \ "" \ diff --git a/tests/test_strix_unverified_dependency.py b/tests/test_strix_unverified_dependency.py new file mode 100644 index 0000000000..a4d2ac8c87 --- /dev/null +++ b/tests/test_strix_unverified_dependency.py @@ -0,0 +1,100 @@ +"""Strix findings naming a package the repository does not depend on are unverified. + +fast-mlsirm#2246 (run 36580588738) failed its required Strix gate on +"VULN-0001: CVE-2024-1234 in lodash 4.17.20" from the free fallback model. The +repository is Rust and Python with no JavaScript lockfile, and CVE-2024-1234 is +unrelated to lodash. With no file location the gate failed closed as unmapped. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +from scripts.ci.strix_unverified_dependency import ( + named_packages, + unverified_dependency_finding, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +HELPER = REPO_ROOT / "scripts/ci/strix_unverified_dependency.py" + +# Verbatim vuln-0001.md from the strix-reports artifact of run 36580588738. +LODASH_REPORT = """# CVE-2024-1234 in lodash 4.17.20 (prototype pollution) + +**ID:** vuln-0001 +**Severity:** MEDIUM +**Found:** 2026-09-29 14:22:11 UTC +**Target:** lodash 4.17.20 +**Package:** lodash +**Ecosystem:** npm +**Installed Version:** 4.17.20 +**Fixed Version:** 4.17.21 +**Introduced By:** express@4.18.1 +**Dependency Chain:** express@4.18.1 > lodash@4.17.20 +**CVE:** CVE-2024-1234 +**CWE:** CWE-78 +**CVSS:** 5.6 +**Fix Effort:** Low + +## Description + +This report documents the vulnerability CVE-2024-1234 in lodash 4.17.20, which allows for prototype pollution. +""" + + +def _rust_python_repo(root: Path) -> Path: + root.mkdir() + (root / "Cargo.lock").write_text('version = 3\n\n[[package]]\nname = "itoa"\nversion = "1.0.11"\n') + (root / "uv.lock").write_text('version = 1\n\n[[package]]\nname = "numpy"\nversion = "2.1.0"\n') + (root / "pyproject.toml").write_text('[project]\nname = "demo"\ndependencies = ["numpy>=2"]\n') + return root + + +def test_package_names_come_only_from_structured_fields() -> None: + assert named_packages(LODASH_REPORT) == {"lodash", "express"} + assert named_packages("**Target:** lodash@4.17.20\n") == {"lodash"} + assert named_packages("**Package:** @babel/core\n") == {"@babel/core"} + assert named_packages("**Target:** crates/core/src/lib.rs:10\n") == set() + assert named_packages("**Target:** src/app.py\n") == set() + # The gate console log boxes the same fields (run 36580588738 gate-console.log). + assert named_packages("│ Target: lodash 4.17.20 │\n") == {"lodash"} + # Free text such as "weak TLS in openssl 1.1.1" never names a package. + assert named_packages("# Weak TLS in openssl 1.1.1\n\n**Severity:** HIGH\n") == set() + + +def test_a_named_dependency_present_anywhere_keeps_the_finding(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + (repo / "package.json").write_text('{"dependencies": {"express": "4.18.1"}}') + assert not unverified_dependency_finding(LODASH_REPORT, repo) + + +def test_lodash_claim_on_rust_python_repo_is_unverified(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + assert unverified_dependency_finding(LODASH_REPORT, repo) + + +def test_package_present_in_a_lockfile_stays_a_finding(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + report = "# VULN-0002: RUSTSEC-2099-0001 in itoa 1.0.11\n\n**Severity:** HIGH\n**Target:** itoa 1.0.11\n" + assert not unverified_dependency_finding(report, repo) + (repo / "package-lock.json").write_text('{"packages": {"node_modules/lodash": {"version": "4.17.20"}}}') + assert not unverified_dependency_finding(LODASH_REPORT, repo) + + +def test_findings_without_a_package_name_are_never_dropped(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + report = "# VULN-0003: SQL injection\n\n**Severity:** HIGH\n**Endpoint:** /api/login\n" + assert not unverified_dependency_finding(report, repo) + + +def test_cli_exit_status_and_message(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + report = tmp_path / "vuln-0001.md" + report.write_text(LODASH_REPORT) + result = subprocess.run([sys.executable, str(HELPER), str(report), str(repo)], capture_output=True, text=True, check=False) + assert result.returncode == 0 + assert "lodash" in result.stderr and "unverified" in result.stderr + (repo / "yarn.lock").write_text('lodash@^4.17.20:\n version "4.17.20"\n') + assert subprocess.run([sys.executable, str(HELPER), str(report), str(repo)], check=False).returncode == 1