diff --git a/scripts/ci/strix_report_scope.py b/scripts/ci/strix_report_scope.py index 44de1c441d..00792b9548 100644 --- a/scripts/ci/strix_report_scope.py +++ b/scripts/ci/strix_report_scope.py @@ -4,11 +4,29 @@ from __future__ import annotations import json +import re import sys from pathlib import Path +def names_changed_path(report: str, changed_path: str) -> bool: + """Return whether the report names ``changed_path`` or a component-boundary suffix of it. + + Scanners abbreviate a scoped file to its basename or a trailing directory + slice; a bare substring would also accept ``test_x.py`` for ``x.py``. + """ + parts = changed_path.split("/") + for start in range(len(parts)): + suffix = re.escape("/".join(parts[start:])) + # Only the full path may follow a slash, i.e. sit under an absolute scan root. + before = r"[\w.-]" if start == 0 else r"[\w./-]" + if re.search(rf"(? None: + """Raise ``ValueError`` unless ``output`` holds one completed report naming a changed path.""" if not output.is_dir() or output.is_symlink(): raise ValueError("scan output directory is missing") runs = [path for path in output.iterdir() if path.is_dir() and not path.is_symlink()] @@ -28,7 +46,7 @@ def validate(output: Path, changed_paths: list[str]) -> None: if metadata.get("status") != "completed" or results.get("scan_completed") is not True or results.get("success") is not True: raise ValueError("scan report is incomplete") report = report_path.read_text(encoding="utf-8") - if not any(path in report for path in changed_paths): + if not any(names_changed_path(report, path) for path in changed_paths): raise ValueError("scan report does not identify a changed source file") diff --git a/tests/test_strix_report_scope.py b/tests/test_strix_report_scope.py index 3ff7964659..c4a2f013f7 100644 --- a/tests/test_strix_report_scope.py +++ b/tests/test_strix_report_scope.py @@ -7,6 +7,14 @@ SCRIPT = Path(__file__).resolve().parents[1] / "scripts/ci/strix_report_scope.py" +COMPLETED_RESULTS = { + "scan_completed": True, + "success": True, + "executive_summary": "No issues found in the changed file.", + "methodology": "Reviewed the changed source file.", + "technical_analysis": "The changed function parses without executing input.", + "recommendations": "Retain static parsing.", +} def test_report_scope_rejects_unrelated_success_and_accepts_scoped_success(tmp_path: Path) -> None: @@ -24,3 +32,86 @@ def test_report_scope_rejects_unrelated_success_and_accepts_scoped_success(tmp_p report.write_text("Assessed python/fast_mlsirm/report.py; no vulnerabilities found.\n", encoding="utf-8") assert subprocess.run(command, capture_output=True).returncode == 0 assert subprocess.run(command[:-1], capture_output=True).returncode == 1 + + +def test_report_scope_accepts_path_suffix_at_component_boundary_only(tmp_path: Path) -> None: + run = tmp_path / "current-scan" + run.mkdir() + (run / "run.json").write_text( + json.dumps({"status": "completed", "scan_results": COMPLETED_RESULTS}), + encoding="utf-8", + ) + report = run / "penetration_test_report.md" + command = [sys.executable, str(SCRIPT), str(tmp_path), "scripts/ci/check_telemetry_ownership.py"] + + report.write_text("Reviewed `check_telemetry_ownership.py`; no findings.\n", encoding="utf-8") + assert subprocess.run(command, capture_output=True).returncode == 0 + report.write_text("Reviewed ci/check_telemetry_ownership.py; no findings.\n", encoding="utf-8") + assert subprocess.run(command, capture_output=True).returncode == 0 + report.write_text("Reviewed /workspace/scope/scripts/ci/check_telemetry_ownership.py.\n", encoding="utf-8") + assert subprocess.run(command, capture_output=True).returncode == 0 + + report.write_text("Reviewed test_check_telemetry_ownership.py; no findings.\n", encoding="utf-8") + assert subprocess.run(command, capture_output=True).returncode == 1 + report.write_text("Reviewed check_telemetry_ownership.pyc; no findings.\n", encoding="utf-8") + assert subprocess.run(command, capture_output=True).returncode == 1 + report.write_text("Reviewed i/check_telemetry_ownership.py; no findings.\n", encoding="utf-8") + assert subprocess.run(command, capture_output=True).returncode == 1 + + +def _load_module(): + import importlib.util + + spec = importlib.util.spec_from_file_location("strix_report_scope", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_report_scope_rejects_each_malformed_report_shape(tmp_path: Path) -> None: + import pytest + + scope = _load_module() + with pytest.raises(ValueError, match="directory is missing"): + scope.validate(tmp_path / "absent", ["a.py"]) + link = tmp_path / "link" + link.symlink_to(tmp_path) + with pytest.raises(ValueError, match="directory is missing"): + scope.validate(link, ["a.py"]) + output = tmp_path / "out" + output.mkdir() + with pytest.raises(ValueError, match="exactly one"): + scope.validate(output, ["a.py"]) + run = output / "run" + run.mkdir() + with pytest.raises(ValueError, match="missing or linked"): + scope.validate(output, ["a.py"]) + metadata = run / "run.json" + (run / "penetration_test_report.md").write_text("a.py\n", encoding="utf-8") + for body, message in ( + ([], "not an object"), + ({"scan_results": [1]}, "results are not an object"), + ({"status": "completed", "scan_results": {"scan_completed": True}}, "incomplete"), + ): + metadata.write_text(json.dumps(body), encoding="utf-8") + with pytest.raises(ValueError, match=message): + scope.validate(output, ["a.py"]) + metadata.write_text( + json.dumps({"status": "completed", "scan_results": COMPLETED_RESULTS}), + encoding="utf-8", + ) + scope.validate(output, ["src/a.py"]) + with pytest.raises(ValueError, match="does not identify"): + scope.validate(output, ["src/b.py"]) + + +def test_report_scope_main_exits_nonzero_on_invalid_scope(tmp_path: Path, monkeypatch, capsys) -> None: + import runpy + + import pytest + + monkeypatch.setattr(sys, "argv", [str(SCRIPT), str(tmp_path / "absent"), "a.py"]) + with pytest.raises(SystemExit) as exit_info: + runpy.run_path(str(SCRIPT), run_name="__main__") + assert exit_info.value.code == 1 + assert "directory is missing" in capsys.readouterr().err