Audit lane: add an adversarial-verify stage (refute each finding vs code) before findings become cards

changelog.d/tsk-x6fzgf-adversarial-verify.md

@@ -0,0 +1,3 @@
+### Added
+
+- An adversarial-verify stage in the static security analysis pipeline: each finding produced by the code analyzers is now re-examined against its source line to refute false positives inside comments, string literals, or known example values before the findings are surfaced to the user.

tests/test_code_analyzer.py

@@ -13,6 +13,7 @@
 
 from tinyagentos.code_analyzer import (
     Finding,
+    adversarial_verify,
     analyze_app_source,
     detect_dangerous_url_scheme,
     detect_dom_xss_sink,
@@ -290,3 +291,62 @@ def test_finding_to_dict_shape(self):
             "line": 3,
             "message": "msg",
         }
+
+
+# --------------------------------------------------------------------------- #
+# adversarial_verify
+# --------------------------------------------------------------------------- #
+
+
+class TestAdversarialVerify:
+    def test_comment_line_is_refuted(self):
+        findings = [Finding("critical", "eval-like-execution", "app.js", 1, "msg")]
+        result = adversarial_verify(findings, {"app.js": "// eval(userInput);"})
+        assert result == []
+
+    def test_block_comment_start_is_refuted(self):
+        findings = [Finding("critical", "eval-like-execution", "app.js", 1, "msg")]
+        result = adversarial_verify(findings, {"app.js": "/* eval(userInput); */"})
+        assert result == []
+
+    def test_string_literal_is_refuted(self):
+        findings = [Finding("critical", "eval-like-execution", "app.js", 1, "msg")]
+        result = adversarial_verify(findings, {"app.js": 'const msg = "eval() is bad";'})
+        assert result == []
+
+    def test_real_code_is_kept(self):
+        findings = [Finding("critical", "eval-like-execution", "app.js", 1, "msg")]
+        result = adversarial_verify(findings, {"app.js": "eval(userInput);"})
+        assert len(result) == 1
+
+    def test_known_example_key_is_refuted(self):
+        findings = [Finding("critical", "hardcoded-secret", "app.js", 1, "msg")]
+        result = adversarial_verify(findings, {"app.js": 'const key = "AKIAIOSFODNN7EXAMPLE";'})
+        assert result == []
+
+    def test_multiple_findings_mixed(self):
+        findings = [
+            Finding("critical", "eval-like-execution", "app.js", 1, "msg"),
+            Finding("critical", "hardcoded-secret", "app.js", 2, "msg"),
+            Finding("critical", "eval-like-execution", "app.js", 3, "msg"),
+        ]
+        files = {
+            "app.js": (
+                "// eval(userInput);\n"
+                'const key = "AKIAIOSFODNN7EXAMPLE";\n'
+                "eval(userInput);\n"
+            ),
+        }
+        result = adversarial_verify(findings, files)
+        assert len(result) == 1
+        assert result[0].line == 3
+
+    def test_unknown_rule_id_is_kept(self):
+        findings = [Finding("critical", "unknown-rule", "app.js", 1, "msg")]
+        result = adversarial_verify(findings, {"app.js": "some code here"})
+        assert len(result) == 1
+
+    def test_missing_file_line_drops_finding(self):
+        findings = [Finding("critical", "eval-like-execution", "app.js", 99, "msg")]
+        result = adversarial_verify(findings, {"app.js": "eval(userInput);"})
+        assert result == []

tinyagentos/code_analyzer.py

@@ -462,16 +462,102 @@ def analyze_app_source(files: dict[str, str]) -> list[Finding]:
     content. Findings are returned in a stable order: file (as given in the
     dict), then line number, then detector registration order -- so repeat
     calls against the same input are deterministic and diffable.
+
+    After the initial detector pass, findings are run through an
+    adversarial-verify stage that refutes false positives by checking
+    whether the detected pattern appears inside a string literal, comment,
+    or known example value.
     """
     findings: list[Finding] = []
     for filename, content in files.items():
         for detector in _ALL_DETECTORS:
             findings.extend(detector(filename, content))
     file_order = {name: idx for idx, name in enumerate(files)}
     findings.sort(key=lambda f: (file_order[f.file], f.line))
-    return findings
+    return adversarial_verify(findings, files)
 
 
 def has_critical(findings: list[Finding]) -> bool:
     """Return True if any finding is severity "critical"."""
     return any(f.severity == "critical" for f in findings)
+
+
+# --------------------------------------------------------------------------- #
+# Adversarial verification
+# --------------------------------------------------------------------------- #
+
+# Trigger tokens used to decide whether a finding's pattern sits inside a
+# string literal.  The token is the shortest unique substring the detector
+# matches on that line.
+_TRIGGER_TOKENS: dict[str, str] = {
+    "eval-like-execution": "eval(",
+    "network-exfil": "fetch(",
+    "dom-xss-sink": ".innerHTML",
+    "dangerous-url-scheme": "javascript:",
+    "inline-event-handler-injection": "setAttribute(",
+    "hardcoded-secret": "AKIA",
+    "sandbox-escape-attempt": "window.parent",
+    "postmessage-no-origin-check": "postMessage(",
+    "storage-exfil": "localStorage.",
+}
+
+# Known example/placeholder values that should never trigger a real finding.
+_KNOWN_EXAMPLES: tuple[str, ...] = (
+    "AKIAIOSFODNN7EXAMPLE",
+)
+
+
+def adversarial_verify(findings: list[Finding], files: dict[str, str]) -> list[Finding]:
+    """Second-pass adversarial check that refutes false-positive findings.
+
+    Each finding is re-examined against its source line.  A finding is
+    dropped when the line context clearly shows the detected pattern is
+    inert -- e.g. it lives inside a string literal, a comment, or matches
+    a known example/placeholder value.
+    """
+    verified: list[Finding] = []
+    for f in findings:
+        content = files.get(f.file, "")
+        if not content:
+            verified.append(f)
+            continue
+        lines = content.splitlines()
+        if f.line < 1 or f.line > len(lines):
+            continue
+        line = lines[f.line - 1]
+        if _is_clearly_false_positive(f, line):
+            continue
+        verified.append(f)
+    return verified
+
+
+def _is_clearly_false_positive(finding: Finding, line: str) -> bool:
+    stripped = line.lstrip()
+
+    if stripped.startswith("//"):
+        return True
+    if stripped.startswith("/*"):
+        return True
+    if stripped.startswith("*") and not stripped.startswith("*/"):
+        return True
+
+    if _trigger_is_inside_string(finding, line):
+        return True
+
+    if finding.rule_id == "hardcoded-secret":
+        for example in _KNOWN_EXAMPLES:
+            if example in line:
+                return True
+
+    return False
+
+
+def _trigger_is_inside_string(finding: Finding, line: str) -> bool:
+    token = _TRIGGER_TOKENS.get(finding.rule_id)
+    if not token:
+        return False
+    idx = line.find(token)
+    if idx == -1:
+        return False
+    before = line[:idx]
+    return before.count('"') % 2 == 1 or before.count("'") % 2 == 1