From e600e14b0d627f24b570c759a098ec884f532603 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 30 Sep 2026 00:53:00 +0900 Subject: [PATCH] fix(strix): record findings on packages the repository never uses as unverified fast-mlsirm#2246's required Strix gate failed (run 36580588738) on a free-model report "CVE-2024-1234 in lodash 4.17.20" introduced by express@4.18.1. fast-mlsirm is Rust/Python with no JavaScript lockfile, and CVE-2024-1234 is not a lodash advisory. The finding had no file location, so the gate failed closed as unmapped. New helper scripts/ci/strix_unverified_dependency.py reads package names only from the report's structured Target/Package/Introduced By fields (file and boxed console-log forms; never free text) and reports the finding unverified only when every named package is absent from every dependency manifest and lockfile. The gate applies it only to threshold findings with no location, never when the pull request changes a dependency manifest (REPO_ROOT is the base checkout), and emits a ::warning:: annotation. The new allow_unverified_dependency decision follows allow_baseline. Tests: 6 unit tests on the verbatim artifact report; 3 gate harness cases (lodash absent passes with the warning; lodash present stays blocked; a PR changing package.json stays blocked), the absent case red before the gate change. Full harness PASS; 205 strix tests pass plain and GITHUB_ACTIONS=true. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PtgtJk1WjqieqvLDb1w8uW --- .../20260930-strix-unverified-dependency.md | 9 ++ scripts/ci/strix_quick_gate.sh | 38 ++++++- scripts/ci/strix_unverified_dependency.py | 98 +++++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 59 +++++++++++ tests/test_strix_unverified_dependency.py | 100 ++++++++++++++++++ 5 files changed, 301 insertions(+), 3 deletions(-) create mode 100644 CHANGELOG.d/20260930-strix-unverified-dependency.md create mode 100755 scripts/ci/strix_unverified_dependency.py create mode 100644 tests/test_strix_unverified_dependency.py diff --git a/CHANGELOG.d/20260930-strix-unverified-dependency.md b/CHANGELOG.d/20260930-strix-unverified-dependency.md new file mode 100644 index 0000000000..44112822d4 --- /dev/null +++ b/CHANGELOG.d/20260930-strix-unverified-dependency.md @@ -0,0 +1,9 @@ +## Changed + +- The Strix gate no longer fails closed on a finding that names only packages + the repository does not depend on. When such a finding has no file location, + every package in its Target, Package and Introduced By fields is absent from + every dependency manifest and lockfile, and the pull request changes no + manifest, the gate records it as unverified with a warning annotation. A + model reported a lodash CVE (itself misattributed) on a Rust/Python + repository with no JavaScript dependencies. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index ccf08f48eb..6586482d8c 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2260,6 +2260,23 @@ vulnerability_file_is_below_threshold() { [ "$report_rank" -ge 0 ] && [ "$report_rank" -lt "$threshold_rank" ] } +# A model can name a CVE in a package the repository never depends on +# (fast-mlsirm#2246: "CVE-2024-1234 in lodash 4.17.20" on a Rust/Python tree). +# Such a finding has no file location; record it as unverified instead of +# failing closed, but only when every package it names is absent from every +# dependency manifest and lockfile, and only when the pull request changes no +# dependency manifest (REPO_ROOT is the base checkout, so a dependency the PR +# adds would otherwise look absent). +vulnerability_file_is_unverified_dependency() { + local changed_file + for changed_file in "${CHANGED_FILES[@]}"; do + if is_dependency_manifest_path "$changed_file"; then + return 1 + fi + done + python3 -I "$SCRIPT_DIR/strix_unverified_dependency.py" "$1" "$REPO_ROOT" +} + evaluate_pull_request_findings() { PR_FINDINGS_DECISION="not_applicable" if ! is_pull_request_event; then @@ -2276,6 +2293,7 @@ evaluate_pull_request_findings() { local found_baseline_threshold_finding=0 local found_changed_manifest_only_threshold_finding=0 local found_retryable_model_inconsistency=0 + local found_unverified_dependency=0 local found_any_vuln_file=0 local run_dir vulnerabilities_dir vuln_file line severity rank for run_dir in "$STRIX_REPORTS_DIR"/*; do @@ -2310,6 +2328,10 @@ evaluate_pull_request_findings() { mapfile -t vulnerability_location_records < <(extract_vulnerability_location_records "$vuln_file") mapfile -t vulnerability_locations < <(extract_vulnerability_locations "$vuln_file") if [ "${#vulnerability_locations[@]}" -eq 0 ]; then + if vulnerability_file_is_unverified_dependency "$vuln_file"; then + found_unverified_dependency=1 + continue + fi PR_FINDINGS_DECISION="block_unmapped" echo "Unable to map Strix findings to changed files; failing closed for pull request." >&2 return 1 @@ -2349,6 +2371,12 @@ evaluate_pull_request_findings() { done done + if [ "$found_baseline_threshold_finding" -eq 0 ] && [ "$found_changed_manifest_only_threshold_finding" -eq 0 ] && + [ "$found_unverified_dependency" -eq 1 ] && vulnerability_file_is_unverified_dependency "$STRIX_LOG"; then + PR_FINDINGS_DECISION="allow_unverified_dependency" + return 0 + fi + if [ "$found_baseline_threshold_finding" -eq 0 ] && [ "$found_changed_manifest_only_threshold_finding" -eq 0 ]; then rank="$(extract_max_severity_rank "$STRIX_LOG")" if [ "$rank" -lt 0 ]; then @@ -2443,6 +2471,9 @@ has_unmapped_threshold_report() { local vulnerability_locations=() mapfile -t vulnerability_locations < <(extract_vulnerability_locations "$vuln_file") if [ "${#vulnerability_locations[@]}" -eq 0 ]; then + if vulnerability_file_is_unverified_dependency "$vuln_file"; then + continue + fi return 0 fi done @@ -2996,7 +3027,8 @@ PY return 1 fi if has_blocking_vulnerability_reports; then - if ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then + if ! evaluate_pull_request_findings || + { [ "$PR_FINDINGS_DECISION" != "allow_baseline" ] && [ "$PR_FINDINGS_DECISION" != "allow_unverified_dependency" ]; }; then echo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 return 1 fi @@ -3653,7 +3685,7 @@ has_blocking_vulnerability_reports() { fail_reported_vulnerabilities_before_fallback_success() { case "$PR_FINDINGS_DECISION" in - allow_baseline) + allow_baseline | allow_unverified_dependency) return 1 ;; esac @@ -4586,7 +4618,7 @@ run_current_target_scan() { fi if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && - [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then + { [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] || [ "$PR_FINDINGS_DECISION" = "allow_unverified_dependency" ]; }; then echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 return 1 fi diff --git a/scripts/ci/strix_unverified_dependency.py b/scripts/ci/strix_unverified_dependency.py new file mode 100755 index 0000000000..bc4c781708 --- /dev/null +++ b/scripts/ci/strix_unverified_dependency.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Classify a Strix finding about a package the repository does not depend on. + +A model can report a CVE in a dependency the scanned repository never uses +(fast-mlsirm#2246: "CVE-2024-1234 in lodash 4.17.20" on a Rust/Python tree). +Such a finding has no file location, so the gate would fail closed as +unmapped. This helper lets the gate record it as unverified instead, but only +when every package the report names is absent from every dependency manifest +and lockfile in the repository. Reports that name no package are never +affected. + +Usage: strix_unverified_dependency.py REPORT REPO_ROOT +Exit 0 when the finding is unverified, 1 otherwise. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +MANIFEST_NAMES = frozenset({ + "Cargo.toml", "Cargo.lock", "pyproject.toml", "uv.lock", "poetry.lock", "Pipfile", "Pipfile.lock", + "setup.cfg", "setup.py", "package.json", "package-lock.json", "npm-shrinkwrap.json", "yarn.lock", + "pnpm-lock.yaml", "go.mod", "go.sum", "Gemfile", "Gemfile.lock", "composer.json", "composer.lock", + "pom.xml", "build.gradle", "build.gradle.kts", "packages.lock.json", "renv.lock", "DESCRIPTION", +}) +SKIP_DIRS = frozenset({".git", "node_modules", "target", ".venv", "venv", "__pycache__", "vendor"}) +MAX_MANIFEST_BYTES = 8 * 1024 * 1024 + +# A package is a bare name or an npm scope (``@scope/name``); anything with a +# path separator otherwise is a file, not a package. +_PACKAGE = r"(@[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*|[A-Za-z0-9][A-Za-z0-9_.-]*)" +_VERSION = r"v?\d+(?:\.\d+)+[A-Za-z0-9.+-]*" +# Report files use "**Target:** x"; the console log boxes the same fields as +# "│ Target: x │". +_FIELD = r"^[ \t]*│?[ \t]*(?:\*\*)?{name}:(?:\*\*)?[ \t]*" +_END = r"[ \t]*│?[ \t]*$" +# Only Strix's structured dependency fields name packages; free text such as +# "weak TLS in openssl 1.1.1" never does. +FIELD_RES = ( + re.compile(_FIELD.format(name="Target") + rf"{_PACKAGE}(?:@|[ \t]+){_VERSION}{_END}", re.MULTILINE), + re.compile(_FIELD.format(name="Package") + rf"{_PACKAGE}(?:(?:@|[ \t]+){_VERSION})?{_END}", re.MULTILINE), + re.compile(_FIELD.format(name="Introduced By") + rf"{_PACKAGE}@{_VERSION}{_END}", re.MULTILINE), +) + + +def named_packages(report: str) -> set[str]: + """Return lower-cased package names from the report's dependency fields.""" + return {m.group(1).lower() for regex in FIELD_RES for m in regex.finditer(report)} + + +def _is_requirements(name: str) -> bool: + return name.startswith("requirements") and name.endswith((".txt", ".in")) + + +def _manifest_text(repo_root: Path) -> str: + chunks = [] + for path in repo_root.rglob("*"): + if any(part in SKIP_DIRS for part in path.relative_to(repo_root).parts[:-1]): + continue + if not path.is_file() or path.is_symlink(): + continue + if path.name not in MANIFEST_NAMES and not _is_requirements(path.name): + continue + if path.stat().st_size <= MAX_MANIFEST_BYTES: + chunks.append(path.read_text(encoding="utf-8", errors="replace").lower()) + return "\n".join(chunks) + + +def unverified_dependency_finding(report: str, repo_root: Path) -> bool: + """True when the report names packages and none appears in any manifest. + + Matching is deliberately loose (``node_modules/lodash`` counts), so doubt + keeps the finding. + """ + packages = named_packages(report) + if not packages: + return False + manifests = _manifest_text(repo_root) + return not any(re.search(rf"(? int: + if len(argv) != 3: + print(__doc__, file=sys.stderr) + return 2 + report = Path(argv[1]).read_text(encoding="utf-8", errors="replace") + if unverified_dependency_finding(report, Path(argv[2])): + names = ", ".join(sorted(named_packages(report))) + print(f"::warning::Strix finding names package(s) {names} absent from every dependency manifest and " + "lockfile; recording it as unverified instead of failing closed.", file=sys.stderr) + return 0 + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 80c4832243..c9468ee401 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -23,6 +23,7 @@ materialize_trusted_gate_fixture() { cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$fixture_script_dir/strix_model_utils.sh" cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$fixture_script_dir/strix_evidence_binding.py" cp "$REPO_ROOT/scripts/ci/strix_report_scope.py" "$fixture_script_dir/strix_report_scope.py" + cp "$REPO_ROOT/scripts/ci/strix_unverified_dependency.py" "$fixture_script_dir/strix_unverified_dependency.py" chmod +x "$fixture_script_dir/strix_quick_gate.sh" } TIMEOUT_TEST_PROCESS_SECONDS="${STRIX_TEST_PROCESS_TIMEOUT_SECONDS:-30}" @@ -3307,6 +3308,9 @@ run_gate_case() { local trusted_script_dir="$tmp_dir/trusted-source/scripts/ci" local gate_under_test="$trusted_script_dir/strix_quick_gate.sh" materialize_trusted_gate_fixture "$trusted_script_dir" + if [ "$scenario" = "pr-unverified-dependency-present" ]; then + printf '{"packages": {"node_modules/lodash": {"version": "4.17.20"}}}\n' >"$repo_root_dir/package-lock.json" + fi if [ "$scenario" = "pr-changed-scope-includes-ci-dependency" ]; then # Consumer source under scan; execution still uses the separate trusted runtime. cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" @@ -4955,6 +4959,35 @@ FINDINGS ;; esac ;; + pr-unverified-dependency-lodash | pr-unverified-dependency-present | pr-unverified-dependency-manifest-changed) + # Verbatim vuln-0001.md from the strix-reports artifact of fast-mlsirm#2246 run 36580588738: a free model reported a CVE in a + # package the Rust/Python repository does not depend on. + mkdir -p "$STRIX_REPORTS_DIR/fake-unverified-dependency/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-unverified-dependency/vulnerabilities/vuln-0001.md" <<'EOS' +# CVE-2024-1234 in lodash 4.17.20 (prototype pollution) + +**ID:** vuln-0001 +**Severity:** MEDIUM +**Found:** 2026-09-29 14:22:11 UTC +**Target:** lodash 4.17.20 +**Package:** lodash +**Ecosystem:** npm +**Installed Version:** 4.17.20 +**Fixed Version:** 4.17.21 +**Introduced By:** express@4.18.1 +**Dependency Chain:** express@4.18.1 > lodash@4.17.20 +**CVE:** CVE-2024-1234 +**CWE:** CWE-78 +**CVSS:** 5.6 +**Fix Effort:** Low +EOS + printf '%s\n' \ + '│ Title: CVE-2024-1234 in lodash 4.17.20 (prototype pollution) │' \ + '│ Severity: MEDIUM │' \ + '│ Target: lodash 4.17.20 │' + echo "Penetration test failed: MEDIUM finding in lodash 4.17.20" + exit 1 + ;; pr-baseline-critical-unchanged) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline/vulnerabilities/vuln-0001.md" <<'EOS' @@ -6166,11 +6199,33 @@ run_github_models_http410_case() { "1" } +run_unverified_dependency_case() { + local scenario="$1" expected_exit expected_message + if [ "$scenario" = "pr-unverified-dependency-lodash" ]; then + expected_exit="0" + expected_message="::warning::Strix finding names package(s) express, lodash absent from every dependency manifest and lockfile; recording it as unverified instead of failing closed." + else + expected_exit="1" + expected_message="Unable to map Strix findings to changed files; failing closed for pull request." + fi + local changed_files="sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" + if [ "$scenario" = "pr-unverified-dependency-manifest-changed" ]; then + # A PR that changes a manifest may add the package; keep failing closed. + changed_files="$changed_files"$'\n'"package.json" + fi + run_gate_case "$scenario" "openai/gpt-4o-mini" "" "$expected_exit" "$expected_message" "1" \ + "openai/gpt-4o-mini" "https://example.invalid" "vertex_ai" "__DEFAULT__" "" "0" "MEDIUM" "0" \ + "" "" "1200" "0" "pull_request" "$changed_files" +} + run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in "") return 0 ;; + pr-unverified-dependency-lodash | pr-unverified-dependency-present | pr-unverified-dependency-manifest-changed) + run_unverified_dependency_case "$STRIX_TEST_CASE_FILTER" + ;; success) run_gate_case "success" \ "vertex_ai/ready-primary" \ @@ -12185,6 +12240,10 @@ run_gate_case "pr-critical-changed" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_unverified_dependency_case pr-unverified-dependency-lodash +run_unverified_dependency_case pr-unverified-dependency-present +run_unverified_dependency_case pr-unverified-dependency-manifest-changed + run_gate_case "pr-changed-file-nonintersecting-line" \ "openai/gpt-4o-mini" \ "" \ diff --git a/tests/test_strix_unverified_dependency.py b/tests/test_strix_unverified_dependency.py new file mode 100644 index 0000000000..a4d2ac8c87 --- /dev/null +++ b/tests/test_strix_unverified_dependency.py @@ -0,0 +1,100 @@ +"""Strix findings naming a package the repository does not depend on are unverified. + +fast-mlsirm#2246 (run 36580588738) failed its required Strix gate on +"VULN-0001: CVE-2024-1234 in lodash 4.17.20" from the free fallback model. The +repository is Rust and Python with no JavaScript lockfile, and CVE-2024-1234 is +unrelated to lodash. With no file location the gate failed closed as unmapped. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +from scripts.ci.strix_unverified_dependency import ( + named_packages, + unverified_dependency_finding, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +HELPER = REPO_ROOT / "scripts/ci/strix_unverified_dependency.py" + +# Verbatim vuln-0001.md from the strix-reports artifact of run 36580588738. +LODASH_REPORT = """# CVE-2024-1234 in lodash 4.17.20 (prototype pollution) + +**ID:** vuln-0001 +**Severity:** MEDIUM +**Found:** 2026-09-29 14:22:11 UTC +**Target:** lodash 4.17.20 +**Package:** lodash +**Ecosystem:** npm +**Installed Version:** 4.17.20 +**Fixed Version:** 4.17.21 +**Introduced By:** express@4.18.1 +**Dependency Chain:** express@4.18.1 > lodash@4.17.20 +**CVE:** CVE-2024-1234 +**CWE:** CWE-78 +**CVSS:** 5.6 +**Fix Effort:** Low + +## Description + +This report documents the vulnerability CVE-2024-1234 in lodash 4.17.20, which allows for prototype pollution. +""" + + +def _rust_python_repo(root: Path) -> Path: + root.mkdir() + (root / "Cargo.lock").write_text('version = 3\n\n[[package]]\nname = "itoa"\nversion = "1.0.11"\n') + (root / "uv.lock").write_text('version = 1\n\n[[package]]\nname = "numpy"\nversion = "2.1.0"\n') + (root / "pyproject.toml").write_text('[project]\nname = "demo"\ndependencies = ["numpy>=2"]\n') + return root + + +def test_package_names_come_only_from_structured_fields() -> None: + assert named_packages(LODASH_REPORT) == {"lodash", "express"} + assert named_packages("**Target:** lodash@4.17.20\n") == {"lodash"} + assert named_packages("**Package:** @babel/core\n") == {"@babel/core"} + assert named_packages("**Target:** crates/core/src/lib.rs:10\n") == set() + assert named_packages("**Target:** src/app.py\n") == set() + # The gate console log boxes the same fields (run 36580588738 gate-console.log). + assert named_packages("│ Target: lodash 4.17.20 │\n") == {"lodash"} + # Free text such as "weak TLS in openssl 1.1.1" never names a package. + assert named_packages("# Weak TLS in openssl 1.1.1\n\n**Severity:** HIGH\n") == set() + + +def test_a_named_dependency_present_anywhere_keeps_the_finding(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + (repo / "package.json").write_text('{"dependencies": {"express": "4.18.1"}}') + assert not unverified_dependency_finding(LODASH_REPORT, repo) + + +def test_lodash_claim_on_rust_python_repo_is_unverified(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + assert unverified_dependency_finding(LODASH_REPORT, repo) + + +def test_package_present_in_a_lockfile_stays_a_finding(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + report = "# VULN-0002: RUSTSEC-2099-0001 in itoa 1.0.11\n\n**Severity:** HIGH\n**Target:** itoa 1.0.11\n" + assert not unverified_dependency_finding(report, repo) + (repo / "package-lock.json").write_text('{"packages": {"node_modules/lodash": {"version": "4.17.20"}}}') + assert not unverified_dependency_finding(LODASH_REPORT, repo) + + +def test_findings_without_a_package_name_are_never_dropped(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + report = "# VULN-0003: SQL injection\n\n**Severity:** HIGH\n**Endpoint:** /api/login\n" + assert not unverified_dependency_finding(report, repo) + + +def test_cli_exit_status_and_message(tmp_path: Path) -> None: + repo = _rust_python_repo(tmp_path / "repo") + report = tmp_path / "vuln-0001.md" + report.write_text(LODASH_REPORT) + result = subprocess.run([sys.executable, str(HELPER), str(report), str(repo)], capture_output=True, text=True, check=False) + assert result.returncode == 0 + assert "lodash" in result.stderr and "unverified" in result.stderr + (repo / "yarn.lock").write_text('lodash@^4.17.20:\n version "4.17.20"\n') + assert subprocess.run([sys.executable, str(HELPER), str(report), str(repo)], check=False).returncode == 1