From 57e5e2fccba6b8bbab572a6ce87d3f003d843ff9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 23:09:12 +0900 Subject: [PATCH 1/6] test(strix): reproduce optional web-search warning false positive --- .../test_strix_optional_web_search_warning.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/test_strix_optional_web_search_warning.py diff --git a/tests/test_strix_optional_web_search_warning.py b/tests/test_strix_optional_web_search_warning.py new file mode 100644 index 0000000000..1578d20a7f --- /dev/null +++ b/tests/test_strix_optional_web_search_warning.py @@ -0,0 +1,95 @@ +"""Regression for Strix's optional web-search capability warning. + +Strix documents PERPLEXITY_API_KEY as optional and its web-search tool returns a +sanitized ``success: false`` result instructing the agent to proceed when the +key is absent. A completed scan must therefore not be reclassified as provider +unavailability solely because that exact warning appears in ``strix.log``. +Unknown warnings remain fail-closed. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" + +OPTIONAL_SEARCH_WARNING = ( + "2026-09-12 12:30:48.087 WARNING strix-pr-scope-vjsusm_b6ed - " + "strix.tools.web_search.tool: web_search invoked without PERPLEXITY_API_KEY configured\n" +) +UNKNOWN_SEARCH_WARNING = ( + "2026-09-12 12:30:48.087 WARNING strix-pr-scope-vjsusm_b6ed - " + "strix.tools.web_search.tool: provider returned malformed search evidence\n" +) +COMPLETION_LINE = ( + "2026-09-12 12:45:56.390 INFO strix-pr-scope-vjsusm_b6ed - " + "strix.core.runner: Strix scan strix-pr-scope-vjsusm_b6ed done\n" +) + + +def _function_block(source: str, function_name: str) -> str: + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _sanitize_then_signal(log_text: str) -> tuple[str, bool]: + gate_source = STRIX_GATE.read_text(encoding="utf-8") + blocks = [ + _function_block(gate_source, name) + for name in ( + "sanitize_known_strix_report_warnings", + "has_strix_report_failure_signal", + ) + ] + with tempfile.TemporaryDirectory(prefix="strix-optional-search-") as temp_dir: + report_root = Path(temp_dir) / "run" + report_root.mkdir() + log_path = report_root / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -uo pipefail", + 'STRIX_REPORTS_DIR="/nonexistent/strix-reports"', + *blocks, + 'sanitize_known_strix_report_warnings "$1"', + 'if has_strix_report_failure_signal "$1"; then echo signal=1; else echo signal=0; fi', + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-optional-search", str(report_root)], + check=False, + capture_output=True, + text=True, + ) + remaining = log_path.read_text(encoding="utf-8") + if completed.returncode != 0: + raise AssertionError(f"rc={completed.returncode}\n{completed.stderr}") + return remaining, "signal=1" in completed.stdout + + +class StrixOptionalWebSearchWarningTests(unittest.TestCase): + def test_missing_optional_perplexity_key_does_not_fail_completed_scan(self) -> None: + remaining, signal = _sanitize_then_signal(OPTIONAL_SEARCH_WARNING + COMPLETION_LINE) + self.assertNotIn("web_search invoked without PERPLEXITY_API_KEY configured", remaining) + self.assertIn("Strix scan strix-pr-scope-vjsusm_b6ed done", remaining) + self.assertFalse(signal) + + def test_unknown_web_search_warning_remains_fail_closed(self) -> None: + remaining, signal = _sanitize_then_signal(UNKNOWN_SEARCH_WARNING + COMPLETION_LINE) + self.assertIn("provider returned malformed search evidence", remaining) + self.assertTrue(signal) + + +if __name__ == "__main__": + unittest.main() From d2d2410092a27b1d82af4ec1857efa657bf5838b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 23:56:33 +0900 Subject: [PATCH 2/6] fix(strix): ignore exact optional search warning --- ...60912-strix-optional-web-search-warning.md | 6 ++++ scripts/ci/strix_quick_gate.sh | 8 +++++ .../test_strix_optional_web_search_warning.py | 29 ++++++++++++++----- 3 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 CHANGELOG.d/20260912-strix-optional-web-search-warning.md diff --git a/CHANGELOG.d/20260912-strix-optional-web-search-warning.md b/CHANGELOG.d/20260912-strix-optional-web-search-warning.md new file mode 100644 index 0000000000..031003b570 --- /dev/null +++ b/CHANGELOG.d/20260912-strix-optional-web-search-warning.md @@ -0,0 +1,6 @@ +### Fixed + +- Keep completed Strix scans valid when their report contains only the exact + upstream warning for missing optional `EXA_API_KEY`, `PERPLEXITY_API_KEY`, or + combined web-search credentials. Unknown warnings remain fail-closed, and raw + report artifacts remain unchanged for audit evidence. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index c08f2fa36c..886c13ec46 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -201,6 +201,13 @@ known_scanner_warning = re.compile( r"^(?:│ MODEL QUALITY WARNING\s+│|" r"Warning: You are sending unauthenticated requests to the HF Hub\.)" ) +known_optional_search_warning = re.compile( + r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ WARNING " + r"[^ ]+ - strix\.tools\.web_search\.tool: " + r"web_search invoked without " + r"(?:EXA_API_KEY|PERPLEXITY_API_KEY|EXA_API_KEY or PERPLEXITY_API_KEY) " + r"configured$" +) def iter_report_logs(root: Path): @@ -231,6 +238,7 @@ for log_path in iter_report_logs(root): for line in lines if not known_internal_warning.match(line) and not known_scanner_warning.match(line) + and not known_optional_search_warning.match(line) ] if filtered != lines: log_path.write_text("".join(filtered), encoding="utf-8") diff --git a/tests/test_strix_optional_web_search_warning.py b/tests/test_strix_optional_web_search_warning.py index 1578d20a7f..9255fa66f2 100644 --- a/tests/test_strix_optional_web_search_warning.py +++ b/tests/test_strix_optional_web_search_warning.py @@ -19,9 +19,10 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" -OPTIONAL_SEARCH_WARNING = ( - "2026-09-12 12:30:48.087 WARNING strix-pr-scope-vjsusm_b6ed - " - "strix.tools.web_search.tool: web_search invoked without PERPLEXITY_API_KEY configured\n" +OPTIONAL_SEARCH_KEY_LABELS = ( + "EXA_API_KEY", + "PERPLEXITY_API_KEY", + "EXA_API_KEY or PERPLEXITY_API_KEY", ) UNKNOWN_SEARCH_WARNING = ( "2026-09-12 12:30:48.087 WARNING strix-pr-scope-vjsusm_b6ed - " @@ -79,11 +80,23 @@ def _sanitize_then_signal(log_text: str) -> tuple[str, bool]: class StrixOptionalWebSearchWarningTests(unittest.TestCase): - def test_missing_optional_perplexity_key_does_not_fail_completed_scan(self) -> None: - remaining, signal = _sanitize_then_signal(OPTIONAL_SEARCH_WARNING + COMPLETION_LINE) - self.assertNotIn("web_search invoked without PERPLEXITY_API_KEY configured", remaining) - self.assertIn("Strix scan strix-pr-scope-vjsusm_b6ed done", remaining) - self.assertFalse(signal) + def test_missing_optional_search_keys_do_not_fail_completed_scan(self) -> None: + for search_key_label in OPTIONAL_SEARCH_KEY_LABELS: + with self.subTest(search_key_label=search_key_label): + optional_search_warning = ( + "2026-09-12 12:30:48.087 WARNING strix-pr-scope-vjsusm_b6ed - " + "strix.tools.web_search.tool: web_search invoked without " + f"{search_key_label} configured\n" + ) + remaining, signal = _sanitize_then_signal( + optional_search_warning + COMPLETION_LINE + ) + self.assertNotIn( + f"web_search invoked without {search_key_label} configured", + remaining, + ) + self.assertIn("Strix scan strix-pr-scope-vjsusm_b6ed done", remaining) + self.assertFalse(signal) def test_unknown_web_search_warning_remains_fail_closed(self) -> None: remaining, signal = _sanitize_then_signal(UNKNOWN_SEARCH_WARNING + COMPLETION_LINE) From 5359a23ef1239bf541c411f8955ddad719dc7ae4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 07:10:37 +0900 Subject: [PATCH 3/6] test(strix): expose inaccurate raw-artifact claim --- tests/test_strix_optional_web_search_warning.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_strix_optional_web_search_warning.py b/tests/test_strix_optional_web_search_warning.py index 9255fa66f2..67ea607a04 100644 --- a/tests/test_strix_optional_web_search_warning.py +++ b/tests/test_strix_optional_web_search_warning.py @@ -18,6 +18,11 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +CHANGELOG_FRAGMENT = ( + REPOSITORY_ROOT + / "CHANGELOG.d" + / "20260912-strix-optional-web-search-warning.md" +) OPTIONAL_SEARCH_KEY_LABELS = ( "EXA_API_KEY", @@ -80,6 +85,11 @@ def _sanitize_then_signal(log_text: str) -> tuple[str, bool]: class StrixOptionalWebSearchWarningTests(unittest.TestCase): + def test_changelog_describes_sanitized_classification_log(self) -> None: + changelog_text = CHANGELOG_FRAGMENT.read_text(encoding="utf-8") + self.assertIn("classification log", changelog_text) + self.assertNotIn("raw report artifacts remain unchanged", changelog_text) + def test_missing_optional_search_keys_do_not_fail_completed_scan(self) -> None: for search_key_label in OPTIONAL_SEARCH_KEY_LABELS: with self.subTest(search_key_label=search_key_label): From 27b915f1111f74d86eb52696680ddf6eb0e4118b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 20 Sep 2026 07:11:15 +0900 Subject: [PATCH 4/6] docs(strix): describe classification-log sanitization --- CHANGELOG.d/20260912-strix-optional-web-search-warning.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.d/20260912-strix-optional-web-search-warning.md b/CHANGELOG.d/20260912-strix-optional-web-search-warning.md index 031003b570..080508b33e 100644 --- a/CHANGELOG.d/20260912-strix-optional-web-search-warning.md +++ b/CHANGELOG.d/20260912-strix-optional-web-search-warning.md @@ -2,5 +2,5 @@ - Keep completed Strix scans valid when their report contains only the exact upstream warning for missing optional `EXA_API_KEY`, `PERPLEXITY_API_KEY`, or - combined web-search credentials. Unknown warnings remain fail-closed, and raw - report artifacts remain unchanged for audit evidence. + combined web-search credentials. Unknown warnings remain fail-closed; only the + trusted classification log is sanitized before failure-signal evaluation. From fb4b3338e6e3af5b88ecbb91ecd3afba0b074fbc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 28 Sep 2026 14:41:29 +0900 Subject: [PATCH 5/6] fix(strix): ignore completed scan PTY threshold notice --- .../20260912-strix-optional-web-search-warning.md | 5 +++-- scripts/ci/strix_quick_gate.sh | 3 ++- tests/test_strix_optional_web_search_warning.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.d/20260912-strix-optional-web-search-warning.md b/CHANGELOG.d/20260912-strix-optional-web-search-warning.md index 080508b33e..361b7032cf 100644 --- a/CHANGELOG.d/20260912-strix-optional-web-search-warning.md +++ b/CHANGELOG.d/20260912-strix-optional-web-search-warning.md @@ -2,5 +2,6 @@ - Keep completed Strix scans valid when their report contains only the exact upstream warning for missing optional `EXA_API_KEY`, `PERPLEXITY_API_KEY`, or - combined web-search credentials. Unknown warnings remain fail-closed; only the - trusted classification log is sanitized before failure-signal evaluation. + combined web-search credentials, or the exact PTY session-count threshold + notice. Unknown warnings remain fail-closed; only the trusted classification log + is sanitized before failure-signal evaluation. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index bd085bccef..1e1ef749f2 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -199,7 +199,8 @@ known_internal_warning = re.compile( ) known_scanner_warning = re.compile( r"^(?:│ MODEL QUALITY WARNING\s+│|" - r"Warning: You are sending unauthenticated requests to the HF Hub\.)" + r"Warning: You are sending unauthenticated requests to the HF Hub\.|" + r"PTY process count reached warning threshold: [0-9]+ active sessions$)" ) known_optional_search_warning = re.compile( r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ WARNING " diff --git a/tests/test_strix_optional_web_search_warning.py b/tests/test_strix_optional_web_search_warning.py index 67ea607a04..1b970163dd 100644 --- a/tests/test_strix_optional_web_search_warning.py +++ b/tests/test_strix_optional_web_search_warning.py @@ -113,6 +113,17 @@ def test_unknown_web_search_warning_remains_fail_closed(self) -> None: self.assertIn("provider returned malformed search evidence", remaining) self.assertTrue(signal) + def test_completed_scan_ignores_only_exact_pty_count_notice(self) -> None: + notice = "PTY process count reached warning threshold: 60 active sessions\n" + remaining, signal = _sanitize_then_signal(notice + COMPLETION_LINE) + self.assertNotIn(notice, remaining) + self.assertFalse(signal) + + unknown = "PTY process count reached warning threshold: unknown active sessions\n" + remaining, signal = _sanitize_then_signal(unknown + COMPLETION_LINE) + self.assertIn(unknown, remaining) + self.assertTrue(signal) + if __name__ == "__main__": unittest.main() From 061cea389aa29ef7d0ce87dde5f59054b797daf2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 28 Sep 2026 15:02:52 +0900 Subject: [PATCH 6/6] test(ci): cover Strix PTY console classification --- .../test_strix_optional_web_search_warning.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_strix_optional_web_search_warning.py b/tests/test_strix_optional_web_search_warning.py index 1b970163dd..dd2df58c05 100644 --- a/tests/test_strix_optional_web_search_warning.py +++ b/tests/test_strix_optional_web_search_warning.py @@ -124,6 +124,42 @@ def test_completed_scan_ignores_only_exact_pty_count_notice(self) -> None: self.assertIn(unknown, remaining) self.assertTrue(signal) + def test_pty_notice_does_not_trigger_console_infrastructure_error(self) -> None: + gate_source = STRIX_GATE.read_text(encoding="utf-8") + blocks = [ + _function_block(gate_source, name) + for name in ( + "sanitize_known_strix_report_warnings", + "has_detected_infrastructure_error", + ) + ] + with tempfile.TemporaryDirectory(prefix="strix-pty-console-") as temp_dir: + log_path = Path(temp_dir) / "console.log" + script = "\n".join( + ( + "set -uo pipefail", + 'STRIX_LOG="$1"', + 'LLM_PROVIDER_ONLY_REGEX="__never__"', + "for name in is_timeout_error is_rate_limit_error is_llm_token_limit_error is_midstream_fallback_error is_llm_api_connection_error is_llm_service_unavailable_error is_nvidia_nim_not_found_error is_model_behavior_error is_caido_bootstrap_timing_error; do eval \"$name() { return 1; }\"; done", + *blocks, + 'sanitize_known_strix_report_warnings "$STRIX_LOG"', + 'if has_detected_infrastructure_error; then echo signal=1; else echo signal=0; fi', + ) + ) + for notice, expected in ( + ("PTY process count reached warning threshold: 60 active sessions\n", False), + ("PTY process count reached warning threshold: unknown active sessions\n", True), + ): + log_path.write_text(notice + COMPLETION_LINE, encoding="utf-8") + completed = subprocess.run( + ["bash", "-c", script, "strix-pty-console", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual("signal=1" in completed.stdout, expected) + if __name__ == "__main__": unittest.main()