Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion scripts/ci/strix_report_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,29 @@
from __future__ import annotations

import json
import re
import sys
from pathlib import Path


def names_changed_path(report: str, changed_path: str) -> bool:
"""Return whether the report names ``changed_path`` or a component-boundary suffix of it.

Scanners abbreviate a scoped file to its basename or a trailing directory
slice; a bare substring would also accept ``test_x.py`` for ``x.py``.
"""
parts = changed_path.split("/")
for start in range(len(parts)):
suffix = re.escape("/".join(parts[start:]))
# Only the full path may follow a slash, i.e. sit under an absolute scan root.
before = r"[\w.-]" if start == 0 else r"[\w./-]"
if re.search(rf"(?<!{before}){suffix}(?![\w/-]|\.\w)", report):
return True
return False


def validate(output: Path, changed_paths: list[str]) -> None:
"""Raise ``ValueError`` unless ``output`` holds one completed report naming a changed path."""
if not output.is_dir() or output.is_symlink():
raise ValueError("scan output directory is missing")
runs = [path for path in output.iterdir() if path.is_dir() and not path.is_symlink()]
Expand All @@ -28,7 +46,7 @@ def validate(output: Path, changed_paths: list[str]) -> None:
if metadata.get("status") != "completed" or results.get("scan_completed") is not True or results.get("success") is not True:
raise ValueError("scan report is incomplete")
report = report_path.read_text(encoding="utf-8")
if not any(path in report for path in changed_paths):
if not any(names_changed_path(report, path) for path in changed_paths):
raise ValueError("scan report does not identify a changed source file")


Expand Down
91 changes: 91 additions & 0 deletions tests/test_strix_report_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@


SCRIPT = Path(__file__).resolve().parents[1] / "scripts/ci/strix_report_scope.py"
COMPLETED_RESULTS = {
"scan_completed": True,
"success": True,
"executive_summary": "No issues found in the changed file.",
"methodology": "Reviewed the changed source file.",
"technical_analysis": "The changed function parses without executing input.",
"recommendations": "Retain static parsing.",
}


def test_report_scope_rejects_unrelated_success_and_accepts_scoped_success(tmp_path: Path) -> None:
Expand All @@ -24,3 +32,86 @@ def test_report_scope_rejects_unrelated_success_and_accepts_scoped_success(tmp_p
report.write_text("Assessed python/fast_mlsirm/report.py; no vulnerabilities found.\n", encoding="utf-8")
assert subprocess.run(command, capture_output=True).returncode == 0
assert subprocess.run(command[:-1], capture_output=True).returncode == 1


def test_report_scope_accepts_path_suffix_at_component_boundary_only(tmp_path: Path) -> None:
run = tmp_path / "current-scan"
run.mkdir()
(run / "run.json").write_text(
json.dumps({"status": "completed", "scan_results": COMPLETED_RESULTS}),
encoding="utf-8",
)
report = run / "penetration_test_report.md"
command = [sys.executable, str(SCRIPT), str(tmp_path), "scripts/ci/check_telemetry_ownership.py"]

report.write_text("Reviewed `check_telemetry_ownership.py`; no findings.\n", encoding="utf-8")
assert subprocess.run(command, capture_output=True).returncode == 0
report.write_text("Reviewed ci/check_telemetry_ownership.py; no findings.\n", encoding="utf-8")
assert subprocess.run(command, capture_output=True).returncode == 0
report.write_text("Reviewed /workspace/scope/scripts/ci/check_telemetry_ownership.py.\n", encoding="utf-8")
assert subprocess.run(command, capture_output=True).returncode == 0

report.write_text("Reviewed test_check_telemetry_ownership.py; no findings.\n", encoding="utf-8")
assert subprocess.run(command, capture_output=True).returncode == 1
report.write_text("Reviewed check_telemetry_ownership.pyc; no findings.\n", encoding="utf-8")
assert subprocess.run(command, capture_output=True).returncode == 1
report.write_text("Reviewed i/check_telemetry_ownership.py; no findings.\n", encoding="utf-8")
assert subprocess.run(command, capture_output=True).returncode == 1


def _load_module():
import importlib.util

spec = importlib.util.spec_from_file_location("strix_report_scope", SCRIPT)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module


def test_report_scope_rejects_each_malformed_report_shape(tmp_path: Path) -> None:
import pytest

scope = _load_module()
with pytest.raises(ValueError, match="directory is missing"):
scope.validate(tmp_path / "absent", ["a.py"])
link = tmp_path / "link"
link.symlink_to(tmp_path)
with pytest.raises(ValueError, match="directory is missing"):
scope.validate(link, ["a.py"])
output = tmp_path / "out"
output.mkdir()
with pytest.raises(ValueError, match="exactly one"):
scope.validate(output, ["a.py"])
run = output / "run"
run.mkdir()
with pytest.raises(ValueError, match="missing or linked"):
scope.validate(output, ["a.py"])
metadata = run / "run.json"
(run / "penetration_test_report.md").write_text("a.py\n", encoding="utf-8")
for body, message in (
([], "not an object"),
({"scan_results": [1]}, "results are not an object"),
({"status": "completed", "scan_results": {"scan_completed": True}}, "incomplete"),
):
metadata.write_text(json.dumps(body), encoding="utf-8")
with pytest.raises(ValueError, match=message):
scope.validate(output, ["a.py"])
metadata.write_text(
json.dumps({"status": "completed", "scan_results": COMPLETED_RESULTS}),
encoding="utf-8",
)
scope.validate(output, ["src/a.py"])
with pytest.raises(ValueError, match="does not identify"):
scope.validate(output, ["src/b.py"])


def test_report_scope_main_exits_nonzero_on_invalid_scope(tmp_path: Path, monkeypatch, capsys) -> None:
import runpy

import pytest

monkeypatch.setattr(sys, "argv", [str(SCRIPT), str(tmp_path / "absent"), "a.py"])
with pytest.raises(SystemExit) as exit_info:
runpy.run_path(str(SCRIPT), run_name="__main__")
assert exit_info.value.code == 1
assert "directory is missing" in capsys.readouterr().err
Loading