alter pre gate to handle multiline secrets

detect_secrets/core/gate.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import re
+from typing import Iterable
+from typing import List
+from typing import Pattern
+from typing import Set
+from typing import Tuple
+from typing import TYPE_CHECKING
+
+from detect_secrets.plugins.keyword import DENYLIST as KEYWORD_DENYLIST
+
+if TYPE_CHECKING:
+    from detect_secrets.plugins.base import BasePlugin
+
+# Real precondition for high-entropy candidate extraction (no length
+# threshold exists in the real extraction regex).
+_ENTROPY_DELIMITER_PATTERN = r'[\'":=]'
+
+# `denylist` is the RegexBasedDetector contract; `multiline_deny_list` is checkov's
+# CustomRegexDetector-specific attribute for isMultiline policies with no prerun.
+# Read both via duck typing so this module and scan.py never need a checkov import.
+_PATTERN_COLLECTION_ATTRS = ('denylist', 'multiline_deny_list')
+
+# Strips named groups so combining patterns that reuse the same group name
+# (e.g. two "begin_key" groups) doesn't raise `redefinition of group name`.
+_NAMED_GROUP_RE = re.compile(r'\(\?P<[^>]+>')
+
+# A global inline flag (e.g. `(?i)`) is only legal as the very first token
+# of a pattern; embedding it inside `(?:...)` raises `re.error`. Converting
+# it to the scoped form `(?i:...)` is legal anywhere and matches the same.
+_LEADING_GLOBAL_FLAGS_RE = re.compile(r'^\(\?([aiLmsux]+)\)')
+
+# Constructs that require crossing a line boundary -- see module docstring.
+_MULTILINE_MARKERS = (r'\n', r'\r', '(?s)')
+
+
+def _strip_named_groups(pattern: str) -> str:
+    return _NAMED_GROUP_RE.sub('(?:', pattern)
+
+
+def _scope_leading_flags(pattern: str) -> str:
+    match = _LEADING_GLOBAL_FLAGS_RE.match(pattern)
+    if not match:
+        return pattern
+    flags = match.group(1)
+    rest = pattern[match.end():]
+    return f'(?{flags}:{rest})'
+
+
+def _line_safe_prefix(pattern: str) -> str:
+    """
+    Cut `pattern` before the first line-crossing construct, backing up
+    further out of any unclosed group/character class. Returns '' if no
+    safe non-empty prefix exists -- callers must treat that as "cannot be
+    reduced," not "matches everything."
+    """
+    cut_at = len(pattern)
+    for marker in _MULTILINE_MARKERS:
+        idx = pattern.find(marker)
+        if idx > -1:
+            cut_at = min(cut_at, idx)
+    prefix = pattern[:cut_at]
+
+    # One stack for both '(' groups and '[' classes (in open order), so a
+    # bracket nested inside a group backs up past the group too -- e.g.
+    # `(?P<x>[A-Za-z\n]{10,})` must drop the whole group, not just `[...]`.
+    stack: List[Tuple[str, int]] = []
+    in_bracket = False
+    i = 0
+    n = len(prefix)
+    while i < n:
+        c = prefix[i]
+        if c == '\\':
+            i += 2
+            continue
+        if in_bracket:
+            if c == ']':
+                in_bracket = False
+                stack.pop()
+        else:
+            if c == '[':
+                in_bracket = True
+                stack.append(('bracket', i))
+            elif c == '(':
+                stack.append(('group', i))
+            elif c == ')':
+                if stack and stack[-1][0] == 'group':
+                    stack.pop()
+        i += 1
+
+    if stack:
+        return prefix[:stack[0][1]]
+    return prefix
+
+
+def _make_combinable(pattern: str) -> str:
+    """Reduce an independently-authored pattern to a fragment safely
+    embeddable inside a larger `(?:...)` alternation."""
+    return _scope_leading_flags(_strip_named_groups(_line_safe_prefix(pattern)))
+
+
+class Gate:
+    """
+    A rebuildable, sound line pre-gate.
+
+    `could_contain_secret(line)` returns False only if no currently loaded
+    plugin could possibly match that line. Patterns that can't be safely
+    combined are checked individually -- see `untriggerable_plugins`.
+    """
+
+    def __init__(self) -> None:
+        self._combined: Pattern[str] | None = None
+        self._standalone: List[Pattern[str]] = []
+        self.untriggerable_plugins: Set[str] = set()
+        self.trigger_pattern_count = 0
+
+    def build(self, plugins: Iterable[BasePlugin]) -> None:
+        """(Re)build the gate from the currently loaded plugin set."""
+        fragments: List[str] = [_make_combinable(word) for word in KEYWORD_DENYLIST]
+        fragments.append(_ENTROPY_DELIMITER_PATTERN)
+
+        standalone: List[Pattern[str]] = []
+        untriggerable: Set[str] = set()
+        for plugin in plugins:
+            plugin_name = type(plugin).__name__
+            for attr_name in _PATTERN_COLLECTION_ATTRS:
+                for compiled in getattr(plugin, attr_name, None) or []:
+                    combinable = _make_combinable(compiled.pattern)
+                    can_combine = combinable and _compiles(f'(?:{combinable})')
+                    if not can_combine:
+                        standalone.append(compiled)
+                        untriggerable.add(plugin_name)
+                        continue
+                    fragments.append(combinable)
+
+        combined_source = '(?i)(?:' + '|'.join(f'(?:{f})' for f in fragments) + ')'
+        try:
+            self._combined = re.compile(combined_source)
+        except re.error:
+            # Defensive fallback: if the union itself fails to compile,
+            # don't silently produce a broken gate -- compile each
+            # fragment on its own instead.
+            self._combined = None
+            standalone.extend(re.compile(f'(?i)(?:{f})') for f in fragments)
+
+        self._standalone = standalone
+        self.untriggerable_plugins = untriggerable
+        self.trigger_pattern_count = len(fragments) + len(standalone)
+
+    def could_contain_secret(self, line: str) -> bool:
+        """True if `line` could possibly match some loaded plugin."""
+        if self._combined is not None and self._combined.search(line):
+            return True
+        return any(p.search(line) for p in self._standalone)
+
+
+def _compiles(pattern: str) -> bool:
+    try:
+        re.compile(pattern)
+        return True
+    except re.error:
+        return False
+
+
+def build_gate(plugins: Iterable[BasePlugin]) -> Gate:
+    gate = Gate()
+    gate.build(plugins)
+    return gate

detect_secrets/core/scan.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-import re
 import subprocess
 from functools import lru_cache
 from typing import Any
@@ -27,6 +26,8 @@
 from ..util.code_snippet import get_code_snippet
 from ..util.inject import call_function_with_arguments
 from ..util.path import get_relative_path
+from .gate import build_gate
+from .gate import Gate
 from .log import log
 from .potential_secret import PotentialSecret
 from detect_secrets.util.filetype import determine_file_type
@@ -44,77 +45,25 @@
 # Cache: maps frozenset(parameters) -> list of matching filter functions.
 # Invalidated by cache_bust() in settings.py via the callback registered below.
 _filter_cache: dict[frozenset[str], List[SelfAwareCallable]] = {}
+_PREGATE_ENABLED: bool = os.getenv('DETECT_SECRETS_PERF_PREGATE', '0') != '0'
+_gate: Gate | None = None
 
-_TRIGGER_PATTERN = re.compile(
-    r'(?i)(?:'
-    # Keyword triggers. IMPORTANT: bare "key" and "pass" are included because the
-    # real KeywordDetector DENYLIST (detect_secrets/plugins/keyword.py) matches many
-    # bare/compound forms (client_key, service_key, account_key, db_pass, _pass, etc.)
-    # that a narrower prefixed-only pattern would miss. This intentionally reduces
-    # the pre-gate's filter rate in exchange for correctness — verified empirically
-    # against the parity oracle.
-    r'password|passwd|pwd|secret|token|key|auth|credential|private|'
-    r'cert|certificate|connection[_\-]?string|'
-    r'contrase|nessus|recaptcha|pass'
-    r'|'
-    # PEM private key header
-    r'-----BEGIN'
-    r'|'
-    # JWT: base64-encoded {"  (eyJ)
-    r'eyJ[A-Za-z0-9]'
-    r'|'
-    # AWS key prefix
-    r'AKIA[0-9A-Z]'
-    r'|'
-    # GitHub tokens
-    r'gh[psotr]_[A-Za-z0-9]'
-    r'|'
-    r'github_pat_'
-    r'|'
-    # Stripe
-    r'(?:sk|pk|rk)_(?:live|test)_'
-    r'|'
-    # Slack
-    r'xox[baprs]-'
-    r'|'
-    # SendGrid
-    r'SG\.[A-Za-z0-9\_-]'
-    r'|'
-    # NPM
-    r'npm_[A-Za-z0-9]'
-    r'|'
-    # High-entropy candidate strings. The real HighEntropyStringsPlugin regex is
-    # `([\'":=])\s*([{charset}]+)([\'"]|$)` — i.e. ANY quoted-or-assigned run of
-    # charset characters, with NO minimum length in the extraction regex itself
-    # (the entropy check happens afterward). Mathematically, a string needs at
-    # least ~9 distinct hex characters to exceed the default hex entropy limit
-    # (3.0), and ~15-20 distinct alnum characters to exceed the base64 limit
-    # (4.5) — so thresholds here are deliberately low to avoid false negatives.
-    r'[0-9a-fA-F]{8,}'
-    r'|'
-    r'[A-Za-z0-9+/\_-]{15,}={0,2}'
-    r'|'
-    # Generic: URL with embedded credentials
-    r'://[^@\s]+:[^@\s]+@'
-    r'|'
-    # Azure storage account key
-    r'AccountKey='
-    r'|'
-    # Bearer token
-    r'Bearer\s+[A-Za-z0-9]'
-    r')',
-)
-
-_PREGATE_ENABLED: bool = os.getenv('DETECT_SECRETS_PERF_PREGATE', '1') != '0'
+
+def _get_gate() -> Gate:
+    global _gate
+    if _gate is None:
+        _gate = build_gate(get_plugins())
+    return _gate
 
 
 def _could_contain_secret(line: str) -> bool:
     """
-    Returns True if the line could possibly contain a secret pattern.
-    This is a cheap pre-filter: if it returns False, no detector can match this line.
-    Correctness is verified empirically by the parity oracle.
+    Returns True if the line could possibly contain a secret pattern. This is
+    a cheap pre-filter: if it returns False, no plugin currently loaded for
+    this scan can match this line. Rebuilt from the real, live plugin set --
+    see detect_secrets.core.gate for how correctness is achieved.
     """
-    return bool(_TRIGGER_PATTERN.search(line))
+    return _get_gate().could_contain_secret(line)
 
 
 @lru_cache(maxsize=1)
@@ -433,11 +382,9 @@ def _process_line_based_plugins(
             # skip lines which have too few or too many none whitespace chars
             continue
 
-        # PRE-GATE: skip lines that cannot possibly match any detector pattern.
-        # This avoids building the code_snippet context window and running all
-        # detectors for lines that are provably clean. The gate pattern is a
-        # superset of all detector trigger patterns — if it returns False, no
-        # detector can match. Verified empirically by the parity oracle.
+        # PRE-GATE: skip lines that cannot possibly match any currently loaded
+        # plugin. This avoids building the code_snippet context window and
+        # running all plugins for lines that are provably clean.
         if _PREGATE_ENABLED and not _could_contain_secret(line):
             continue
 
@@ -610,4 +557,10 @@ def _bust_filter_cache() -> None:
     _filter_cache.clear()
 
 
+def _bust_gate_cache() -> None:
+    global _gate
+    _gate = None
+
+
 _settings.register_cache_bust_callback(_bust_filter_cache)
+_settings.register_cache_bust_callback(_bust_gate_cache)