diff --git a/openhack/agents/coordinator.py b/openhack/agents/coordinator.py index ab492e3..98fa211 100644 --- a/openhack/agents/coordinator.py +++ b/openhack/agents/coordinator.py @@ -39,7 +39,7 @@ from openhack.framework_detection import detect_frameworks from openhack.quality import run_quality_gates # Static validator removed — line number correction in hunter, semantic validation by LLM -from openhack.config import settings +from openhack.config import settings, resolve_validator_model, check_self_validation logger = logging.getLogger(__name__) @@ -72,7 +72,15 @@ def get_system_prompt(self, context: dict) -> str: def _create_llm_for_agent(self, agent_type: str) -> LLMClient: model_override = getattr(settings, f"{agent_type}_model_id", None) - model = model_override or self.llm.model + if agent_type == "validator": + hunter_model = getattr(settings, "hunter_model_id", None) or self.llm.model + model = resolve_validator_model( + hunter_model=hunter_model, + requested_validator_model=model_override, + provider=self.llm.provider, + ) + else: + model = model_override or self.llm.model return LLMClient(model=model, temperature=0.0, max_tokens=8192, provider=self.llm.provider, prompt_cache_key=self.llm.prompt_cache_key) @staticmethod @@ -790,7 +798,10 @@ async def run_pass2(task_text, batch_idx): if potential_findings: self.session.add_trace(agent="coordinator", event_type="step_start", content="Step 3: Validation (swarm)") validator_llm = self._create_llm_for_agent("validator") - validator_swarm = ValidatorSwarmAgent(validator_llm, self.tools, self.session) + hunter_model = getattr(settings, "hunter_model_id", None) or self.llm.model + validator_swarm = ValidatorSwarmAgent( + validator_llm, self.tools, self.session, hunter_model=hunter_model + ) validator_result = await validator_swarm.run( "Validate each potential vulnerability.", context=self.context) self.context["validator"] = validator_result diff --git a/openhack/agents/validator.py b/openhack/agents/validator.py index 1195c7a..2f9c886 100644 --- a/openhack/agents/validator.py +++ b/openhack/agents/validator.py @@ -68,16 +68,20 @@ def get_system_prompt(self, context: dict) -> str: findings_text = "" for i, f in enumerate(findings, 1): findings_text += f""" -### Finding {i} +### Candidate Finding {i} (UNVERIFIED HYPOTHESIS) - **Category**: {f.get('category', 'Unknown')} -- **Severity**: {f.get('severity', 'Unknown')} +- **Reported Severity**: {f.get('severity', 'Unknown')} - **File**: {f.get('file_path', 'Unknown')} - **Line**: {f.get('line_number', 'Unknown')} -- **Description**: {f.get('description', 'No description')} -- **Code**: +- **Description / Claim**: {f.get('description', 'No description')} +- **Code Snippet**: ``` {f.get('code_snippet', 'No code snippet')} ``` +> Note: Treat this candidate finding as an unverified hypothesis from the hunter swarm. +> Independently audit the surrounding code, data flow, and mitigations. If the issue is mitigated, +> uses framework auto-escaping, lacks external entity resolution, relies on SameSite cookies, +> or is a framework dev default, mark it as false_positive. """ base_prompt = VALIDATOR_PROMPT.format( diff --git a/openhack/agents/validator_swarm.py b/openhack/agents/validator_swarm.py index 48e5041..6c08e62 100644 --- a/openhack/agents/validator_swarm.py +++ b/openhack/agents/validator_swarm.py @@ -10,7 +10,7 @@ from .llm import LLMClient from .session import Session from openhack.tools.registry import ToolRegistry -from openhack.config import settings +from openhack.config import settings, resolve_validator_model, check_self_validation logger = logging.getLogger(__name__) @@ -19,18 +19,48 @@ class ValidatorSwarmAgent: name = "validator_swarm" description = "Validator swarm coordinator" - def __init__(self, llm: LLMClient, tools: ToolRegistry, session: Session): + def __init__( + self, + llm: LLMClient, + tools: ToolRegistry, + session: Session, + hunter_model: Optional[str] = None, + ): self.llm = llm self.tools = tools self.session = session + self.hunter_model = hunter_model self.total_cost: float = 0.0 self.total_tokens: int = 0 self.total_input_tokens: int = 0 self.total_output_tokens: int = 0 + def _get_models(self) -> tuple[str, str]: + hunter_model = self.hunter_model or getattr(settings, "hunter_model_id", None) or self.llm.model + if settings.validator_model_id: + validator_model = settings.validator_model_id + elif self.llm.model and self.llm.model != hunter_model: + # Caller passed an already-resolved validator client distinct from the hunter model + validator_model = self.llm.model + else: + # Caller passed a generic/default client without resolving an independent validator model; + # preserve automatic cross-model selection so hunter does not review its own findings + validator_model = resolve_validator_model( + hunter_model=hunter_model, + requested_validator_model=None, + provider=self.llm.provider, + ) + return hunter_model, validator_model + def _create_llm_for_sub_validator(self) -> LLMClient: - model = settings.validator_model_id or self.llm.model - return LLMClient(model=model, temperature=0.0, max_tokens=8192, provider=self.llm.provider, prompt_cache_key=self.llm.prompt_cache_key) + _, validator_model = self._get_models() + return LLMClient( + model=validator_model, + temperature=0.0, + max_tokens=8192, + provider=self.llm.provider, + prompt_cache_key=self.llm.prompt_cache_key, + ) def _build_sub_context(self, finding: dict, full_context: dict) -> dict: return { @@ -44,32 +74,69 @@ async def run(self, task: str, context: Optional[dict] = None) -> dict: findings = context.get("hunter", {}).get("findings", []) if not findings: - return {"raw_output": "No findings to validate", "validated_findings": [], "false_positives": [], "type": "validation_complete"} + return { + "raw_output": "No findings to validate", + "validated_findings": [], + "false_positives": [], + "unreviewed": [], + "type": "validation_complete", + } + + hunter_model, validator_model = self._get_models() + + # Startup check: warn if validator uses the same model as hunter (self-validation) + self_val_warning = check_self_validation(hunter_model, validator_model) + if self_val_warning: + logger.warning(self_val_warning) + self.session.add_trace( + agent=self.name, + event_type="warning", + content={ + "warning": "self_validation_detected", + "hunter_model": hunter_model, + "validator_model": validator_model, + "message": self_val_warning, + }, + ) - self.session.add_trace(agent=self.name, event_type="swarm_start", content={"findings_count": len(findings)}) + votes = max(1, getattr(settings, "validator_votes", 1)) + self.session.add_trace( + agent=self.name, + event_type="swarm_start", + content={ + "findings_count": len(findings), + "hunter_model": hunter_model, + "validator_model": validator_model, + "validator_votes": votes, + }, + ) - sub_validators: list[tuple[int, ValidatorAgent, dict]] = [] + sub_validators: list[tuple[int, int, ValidatorAgent, dict]] = [] for idx, finding in enumerate(findings): - llm = self._create_llm_for_sub_validator() - validator = ValidatorAgent(llm, self.tools, self.session, original_finding_index=idx) sub_context = self._build_sub_context(finding, context) - sub_validators.append((idx, validator, sub_context)) + for vote_idx in range(votes): + llm = self._create_llm_for_sub_validator() + validator = ValidatorAgent(llm, self.tools, self.session, original_finding_index=idx) + sub_validators.append((idx, vote_idx, validator, sub_context)) semaphore = asyncio.Semaphore(settings.max_concurrent_validators) - async def run_sub_validator(finding_idx, validator, sub_context): + async def run_sub_validator(finding_idx, vote_idx, validator, sub_context): async with semaphore: try: - sub_task = "Validate this potential vulnerability. Confirm whether it is real, generate a PoC, and suggest a fix." + sub_task = ( + "Validate this potential vulnerability. Confirm whether it is real, " + "generate a PoC, and suggest a fix." + ) result = await validator.run(sub_task, sub_context) - return finding_idx, result + return finding_idx, vote_idx, result except Exception as e: - logger.error(f"Sub-validator for finding {finding_idx} failed: {e}") - return finding_idx, {"validated_findings": [], "false_positives": [], "type": "validation_failed"} + logger.error(f"Sub-validator for finding {finding_idx} (vote {vote_idx}) failed: {e}") + return finding_idx, vote_idx, {"validated_findings": [], "false_positives": [], "type": "validation_failed"} tasks = [ - asyncio.create_task(run_sub_validator(idx, validator, sub_ctx)) - for idx, validator, sub_ctx in sub_validators + asyncio.create_task(run_sub_validator(idx, vote_idx, validator, sub_ctx)) + for idx, vote_idx, validator, sub_ctx in sub_validators ] try: results = await asyncio.gather(*tasks) @@ -79,28 +146,125 @@ async def run_sub_validator(finding_idx, validator, sub_context): await asyncio.gather(*tasks, return_exceptions=True) raise + results_by_finding: dict[int, list[dict]] = {idx: [] for idx in range(len(findings))} + for finding_idx, vote_idx, result in results: + results_by_finding[finding_idx].append(result) + all_validated: list[dict] = [] all_false_positives: list[dict] = [] + all_unreviewed: list[dict] = [] + threshold = (votes // 2) + 1 + + for finding_idx, run_results in results_by_finding.items(): + confirmed_runs = [ + r for r in run_results if r.get("validated_findings") + ] + confirmed_count = len(confirmed_runs) + rejection_runs = [ + r for r in run_results if r.get("false_positives") + ] + failed_runs = [ + r for r in run_results if r.get("type") == "validation_failed" + ] - for finding_idx, result in results: - all_validated.extend(result.get("validated_findings", [])) - all_false_positives.extend(result.get("false_positives", [])) + # Strict majority required (>= threshold). Ties are NOT confirmed. + if confirmed_count >= threshold and confirmed_runs: + best_run = max( + confirmed_runs, + key=lambda r: len(r["validated_findings"][0].get("evidence", "")) + if r["validated_findings"] + else 0, + ) + all_validated.extend(best_run.get("validated_findings", [])) + self.session.add_trace( + agent=self.name, + event_type="validation_consensus", + content={ + "finding_index": finding_idx, + "status": "confirmed", + "confirmed_votes": confirmed_count, + "total_votes": votes, + "threshold": threshold, + }, + ) + elif (len(failed_runs) > 0 and len(rejection_runs) < threshold) or not run_results: + # Review is unfinished or inconclusive because one or more votes failed execution + # (API error, prompt error, or timeout) without achieving a definitive rejection. + # Record unfinished/failed review separately from an actual rejection. + unreviewed_entry = { + "original_index": finding_idx, + "status": "unreviewed", + "confidence": "none", + "evidence": ( + f"Validation review was unfinished or inconclusive ({confirmed_count} confirmed, " + f"{len(rejection_runs)} rejected, {len(failed_runs)} failed with execution errors across {votes} attempts)." + ), + } + all_unreviewed.append(unreviewed_entry) + self.session.add_trace( + agent=self.name, + event_type="validation_consensus", + content={ + "finding_index": finding_idx, + "status": "unreviewed", + "confirmed_votes": confirmed_count, + "rejected_votes": len(rejection_runs), + "failed_votes": len(failed_runs), + "total_votes": votes, + "threshold": threshold, + "review_failed": True, + }, + ) + else: + fp_entry = None + for r in run_results: + if r.get("false_positives"): + fp_entry = r["false_positives"][0] + break + if not fp_entry: + fp_entry = { + "original_index": finding_idx, + "status": "false_positive", + "confidence": "medium", + "evidence": f"Failed strict majority validation ({confirmed_count}/{votes} confirmed votes, required >={threshold})", + } + all_false_positives.append(fp_entry) + self.session.add_trace( + agent=self.name, + event_type="validation_consensus", + content={ + "finding_index": finding_idx, + "status": fp_entry.get("status", "false_positive"), + "confirmed_votes": confirmed_count, + "total_votes": votes, + "threshold": threshold, + "review_failed": False, + }, + ) - for _, validator, _ in sub_validators: + for _, _, validator, _ in sub_validators: self.total_cost += validator.llm.total_cost self.total_tokens += validator.llm.total_tokens self.total_input_tokens += validator.llm.total_input_tokens self.total_output_tokens += validator.llm.total_output_tokens self.session.add_trace( - agent=self.name, event_type="swarm_complete", - content={"total_confirmed": len(all_validated), "total_false_positives": len(all_false_positives), - "total_cost": self.total_cost, "total_tokens": self.total_tokens}, + agent=self.name, + event_type="swarm_complete", + content={ + "total_confirmed": len(all_validated), + "total_false_positives": len(all_false_positives), + "total_unreviewed": len(all_unreviewed), + "total_cost": self.total_cost, + "total_tokens": self.total_tokens, + "validator_votes": votes, + }, ) return { - "raw_output": f"Validated {len(findings)} findings: {len(all_validated)} confirmed, {len(all_false_positives)} false positives", + "raw_output": f"Validated {len(findings)} findings: {len(all_validated)} confirmed, {len(all_false_positives)} false positives, {len(all_unreviewed)} unreviewed", "validated_findings": all_validated, "false_positives": all_false_positives, + "unreviewed": all_unreviewed, "type": "validation_complete", } diff --git a/openhack/config.py b/openhack/config.py index 5b5f96c..76928ed 100644 --- a/openhack/config.py +++ b/openhack/config.py @@ -140,6 +140,9 @@ class Settings(BaseSettings): validator_model_id: Optional[str] = None browser_verifier_model_id: Optional[str] = None + # Number of validation votes per finding. 1 = single pass. Set to 3 for strict >=2-of-3 majority. + validator_votes: int = 1 + max_concurrent_hunters: int = 3 max_concurrent_validators: int = 5 @@ -257,3 +260,46 @@ def reload_settings() -> None: fresh = _build_settings() for field_name in Settings.model_fields: setattr(settings, field_name, getattr(fresh, field_name)) + + +# Cross-model validation defaults: mapping hunter model to an independent validator model +DEFAULT_CROSS_MODELS = { + "gpt-5.6-sol": "kimi-k2.5", + "gpt-5-mini": "kimi-k2.5", + "glm-5.2": "kimi-k2.5", + "glm-5.3-flash": "kimi-k2.5", + "kimi-k2.5": "gpt-5.6-sol", + "claude-sonnet-5": "gpt-5.6-sol", + "meta-llama/Llama-3.3-70B-Instruct": "gpt-5.6-sol", +} + + +def resolve_validator_model( + hunter_model: str, + requested_validator_model: Optional[str] = None, + provider: str = "openhack", +) -> str: + """Resolve validator model, preferring cross-model validation over self-validation.""" + if requested_validator_model: + return requested_validator_model + if provider == "openhack": + # Hosted OpenHack has access to cross-model targets (e.g. K2.5 vs GPT-5.6 / GLM) + return DEFAULT_CROSS_MODELS.get( + hunter_model, + "kimi-k2.5" if hunter_model != "kimi-k2.5" else "gpt-5.6-sol", + ) + return hunter_model + + +def check_self_validation(hunter_model: str, validator_model: str) -> Optional[str]: + """Return warning message if validator model is identical to hunter model.""" + if ( + hunter_model + and validator_model + and hunter_model.strip().lower() == validator_model.strip().lower() + ): + return ( + f"Self-validation detected: validator using same model as hunter ({validator_model}). " + "Cross-model validation recommended to prevent precision inflation." + ) + return None diff --git a/openhack/prompts/validator.py b/openhack/prompts/validator.py index 678b34c..c9c201e 100644 --- a/openhack/prompts/validator.py +++ b/openhack/prompts/validator.py @@ -155,7 +155,7 @@ **B. Browser & Protocol Constraints -- Does the platform actually permit this?** - CORS: `Access-Control-Allow-Origin: *` with `Access-Control-Allow-Credentials: true` is REJECTED by all browsers per the Fetch specification. If the finding relies on this combination, mark it as FALSE POSITIVE. -- CSRF: If the target uses `SameSite=Lax` cookies (browser default) or requires custom headers/tokens that cross-site requests cannot provide (Turnstile tokens, API keys in headers), mark CSRF as FALSE POSITIVE. +- CSRF: Confirm ONLY if the auth cookie is NOT `SameSite=Lax` or `SameSite=Strict`. You MUST read the cookie-setting code (`Set-Cookie`, session options, auth middleware). Modern browsers default cookies to `SameSite=Lax`. If the session cookie is `SameSite=Lax`/`Strict` or lacks cross-site delivery on state-changing requests (POST/PUT/PATCH/DELETE), mark as FALSE POSITIVE. Also mark FALSE POSITIVE if the endpoint requires custom headers (e.g. `Authorization`, `X-CSRF-Token`, custom JSON Content-Type requiring preflight) or bot tokens (Turnstile). - SameSite=None: If intentionally set for embed/widget functionality (a product requirement), this is BY DESIGN, not a vulnerability. **C. Existing Mitigations -- What defenses are in place?** @@ -163,16 +163,22 @@ - Are there downstream authorization checks (e.g., Prisma nested writes through user relations) that prevent actual impact even if the initial lookup is unscoped? - Is there middleware, WAF, or infrastructure-level protection (HSTS at CDN, CSP via headers middleware)? -**D. Damage Assessment -- Is the impact meaningful?** +**D. Parser & Sink Constraints -- Does the engine actually execute or expand this?** +- **XXE (XML External Entity):** Confirm ONLY if the parser explicitly resolves EXTERNAL entities. Popular JavaScript/Node parsers (`xml2js`, `fast-xml-parser`, `sax`, `xml-js`) and safe Python parsers (`defusedxml`) do NOT resolve external entities by default. Unless external entity expansion is explicitly enabled in code configuration (`resolveEntities: true`, `load_dtd=True`, `noent=True`), mark as FALSE POSITIVE. +- **XSS (Cross-Site Scripting):** Confirm ONLY if user-controlled input reaches an executable sink: a RAW-HTML sink (e.g., `dangerouslySetInnerHTML`, `v-html`, `innerHTML`, `document.write`, or unescaped template tags `{{{{ }}}}`), an executable URL attribute (e.g., `href` or `src` accepting `javascript:` pseudo-protocols), or a dynamic script context. Standard JSX/TSX text expressions (e.g., `
{{userInput}}
`) are automatically HTML-escaped (auto-escaped) by React/Next.js/Vue and CANNOT execute script tags; mark those as FALSE POSITIVE. +- **Framework Default Configs:** Framework default development configurations and diagnostic endpoints (e.g., Nuxt DevTools, Next.js telemetry, Vite HMR, Swagger UI in dev mode) are NOT production vulnerabilities unless active and reachable in production builds with real sensitive impact. Mark as FALSE POSITIVE. +- **Blank / Vague Rationale:** If a candidate finding has a blank description, missing code snippet, or a vague tautological claim ("input is not validated" without demonstrating an exploitable sink and payload), mark as FALSE POSITIVE. + +**E. Damage Assessment -- Is the impact meaningful?** - "Attacker can trigger a password reset email" = NOT damage. The link goes to the legitimate user's email inbox. - "Attacker can create a booking" on a scheduling app = NOT damage. Public booking is the product's core function. - "Attacker can determine a UUID exists" = NOT damage. No sensitive information is revealed. - "Attacker can view source code" of an open-source project = NOT damage. The code is already public on GitHub. - "Missing security headers" without a companion injection vulnerability = NOT exploitable damage. -**E. Design Intent -- Is this behavior intentional? ("Too good to be true" check)** +**F. Design Intent -- Is this behavior intentional? ("Too good to be true" check)** -This is the most important criterion. Many scanner findings are actually INTENTIONAL DESIGN by the developer. Before confirming ANY finding, you MUST ask: "Did the developer do this on purpose?" +This is a critical criterion. Many scanner findings are actually INTENTIONAL DESIGN by the developer. Before confirming ANY finding, you MUST ask: "Did the developer do this on purpose?" **Signals of intentional design (finding is likely FALSE POSITIVE):** - The code has a `@since` version tag (versioned, reviewed API — not accidental) @@ -189,9 +195,13 @@ - Is the configuration required for product functionality (embeds, widgets, public APIs, third-party integrations)? - Would "fixing" the finding break core product functionality? +**Definition of `false_positive` vs `confirmed`:** +- `false_positive` = MITIGATED, NOT PRESENT, NOT REACHABLE, or UNEXPLOITABLE. +- Do NOT mark a real, reproducible vulnerability as `false_positive` merely because it is low-severity or low-impact. If the vulnerability is real and exploitable but has minor impact, mark it as `confirmed` with `severity="low"` (and appropriate CVSS score). + **Decision Matrix:** -- Fails ANY of A-E → `false_positive` (with explanation of which criterion failed) -- Passes all A-E with clear evidence → `confirmed` +- Fails ANY of A-F → `false_positive` (with explanation of which criterion failed) +- Passes all A-F with clear evidence → `confirmed` - Uncertain on one criterion → `needs_more_info` ## Potential Vulnerabilities to Validate @@ -308,9 +318,14 @@ **For runtime findings:** A 200 status code with 0 rows is NOT a vulnerability. A 204 on a dummy UUID is NOT a confirmed write. Only ACTUAL DATA in the response proves an exploit. -**For static findings:** A code pattern match is NOT a confirmed vulnerability. You MUST evaluate practical exploitability using criteria A-E above. Key automatic disqualifiers: +**For static findings:** A code pattern match is NOT a confirmed vulnerability. You MUST evaluate practical exploitability using criteria A-F above. Key automatic disqualifiers: - IDOR where IDs are UUIDs/CUIDs → FALSE POSITIVE (cannot guess 122-bit random IDs) - CORS `*` + credentials: true → FALSE POSITIVE (browsers reject this per Fetch spec) +- CSRF where cookie is SameSite=Lax/Strict (or modern browser default) or requires custom headers/tokens → FALSE POSITIVE +- XXE where parser does not explicitly resolve external entities (e.g. xml2js, sax, fast-xml-parser, defusedxml) → FALSE POSITIVE +- XSS where input only reaches standard JSX/TSX text without raw-HTML or executable sink → FALSE POSITIVE +- Framework default dev configuration (Nuxt devtools, Next.js telemetry, Vite HMR) → FALSE POSITIVE +- Blank or generic rationale without verified attacker-controlled sink/payload → FALSE POSITIVE - CSRF on forgot-password/signup/booking endpoints → FALSE POSITIVE (by-design public, no meaningful damage) - Missing headers without companion injection → FALSE POSITIVE (informational only) - Source maps on open-source projects → FALSE POSITIVE (code already public) diff --git a/scripts/run_feature_hunt.py b/scripts/run_feature_hunt.py index 8487bd6..b90feaa 100644 --- a/scripts/run_feature_hunt.py +++ b/scripts/run_feature_hunt.py @@ -32,7 +32,7 @@ from openhack.scan_session import ScanSession from openhack.quality import run_quality_gates from openhack.categories import normalize_category, normalize_severity -from openhack.config import reload_settings, settings +from openhack.config import reload_settings, resolve_validator_model, settings logging.basicConfig( level=logging.INFO, @@ -150,23 +150,43 @@ async def run(target_dir: str, resume_session_id: str = None): # ── Step 5: LLM Validation ────────────────────────────────── print(f"\n[5/5] Validating {len(feature_findings)} findings...") - validator_llm = LLMClient(provider=settings.llm_provider, temperature=0.0, max_tokens=8192) + feature_hunter_model = ( + getattr(settings, "feature_hunter_model_id", None) + or getattr(settings, "hunter_model_id", None) + or coordinator.llm.model + ) + validator_model = resolve_validator_model( + hunter_model=feature_hunter_model, + requested_validator_model=settings.validator_model_id, + provider=settings.llm_provider, + ) + validator_llm = LLMClient( + model=validator_model, + provider=settings.llm_provider, + temperature=0.0, + max_tokens=8192, + ) validator_context = { "hunter": {"findings": feature_findings}, "recon": recon_result, } - validator_swarm = ValidatorSwarmAgent(validator_llm, tools, session) + validator_swarm = ValidatorSwarmAgent( + validator_llm, tools, session, hunter_model=feature_hunter_model + ) validator_result = await validator_swarm.run( "Validate each potential vulnerability.", context=validator_context, ) validated = validator_result.get("validated_findings", []) + false_positives = validator_result.get("false_positives", []) + unreviewed = validator_result.get("unreviewed", []) validator_cost = validator_swarm.total_cost total_cost += validator_cost total_tokens += validator_swarm.total_tokens print(f" Validation complete: {len(validated)} confirmed, " - f"${validator_cost:.4f}") + f"{len(false_positives)} false positives, " + f"{len(unreviewed)} unreviewed, ${validator_cost:.4f}") # Deduplicate and cap validated = CoordinatorAgent._deduplicate_validated(validated, feature_findings) diff --git a/tests/test_cross_model_validation.py b/tests/test_cross_model_validation.py new file mode 100644 index 0000000..fb663c0 --- /dev/null +++ b/tests/test_cross_model_validation.py @@ -0,0 +1,347 @@ +""" +Tests for P0 Cross-Model Validation, Self-Validation Defense, and Consensus Voting. +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch +import pytest + +from openhack.config import ( + Settings, + check_self_validation, + resolve_validator_model, + settings, +) +from openhack.agents.validator import ValidatorAgent +from openhack.agents.validator_swarm import ValidatorSwarmAgent +from openhack.prompts.validator import VALIDATOR_PROMPT + + +class TestCrossModelResolution: + def test_default_cross_model_mapping_openhack(self): + # On OpenHack provider, hunter gpt-5.6-sol should cross-validate with kimi-k2.5 + val_model = resolve_validator_model( + hunter_model="gpt-5.6-sol", + requested_validator_model=None, + provider="openhack", + ) + assert val_model == "kimi-k2.5" + + # Hunter kimi-k2.5 should cross-validate with gpt-5.6-sol + val_model_k25 = resolve_validator_model( + hunter_model="kimi-k2.5", + requested_validator_model=None, + provider="openhack", + ) + assert val_model_k25 == "gpt-5.6-sol" + + def test_explicit_validator_model_override_wins(self): + val_model = resolve_validator_model( + hunter_model="gpt-5.6-sol", + requested_validator_model="claude-sonnet-5", + provider="openhack", + ) + assert val_model == "claude-sonnet-5" + + def test_self_validation_check_detects_same_model(self): + msg = check_self_validation("gpt-5.6-sol", "gpt-5.6-sol") + assert msg is not None + assert "Self-validation detected" in msg + assert "gpt-5.6-sol" in msg + + # Case-insensitive match + msg_upper = check_self_validation("GPT-5.6-SOL", "gpt-5.6-sol") + assert msg_upper is not None + assert "Self-validation detected" in msg_upper + + def test_self_validation_check_returns_none_for_different_models(self): + msg = check_self_validation("gpt-5.6-sol", "kimi-k2.5") + assert msg is None + + +class TestValidatorSwarmConsensus: + @pytest.fixture + def mock_session(self): + session = MagicMock() + session.add_trace = MagicMock() + session.cancelled = False + return session + + @pytest.fixture + def mock_tools(self): + return MagicMock() + + @pytest.fixture + def mock_llm(self): + llm = MagicMock() + llm.model = "gpt-5.6-sol" + llm.provider = "openhack" + llm.prompt_cache_key = None + llm.total_cost = 0.01 + llm.total_tokens = 500 + llm.total_input_tokens = 400 + llm.total_output_tokens = 100 + return llm + + @pytest.mark.asyncio + async def test_self_validation_warning_logged_when_models_equal(self, mock_llm, mock_tools, mock_session): + with patch.object(settings, "validator_model_id", "gpt-5.6-sol"), \ + patch.object(settings, "hunter_model_id", "gpt-5.6-sol"), \ + patch("openhack.agents.validator_swarm.ValidatorAgent.run", new_callable=AsyncMock) as mock_run: + mock_run.return_value = {"validated_findings": [], "false_positives": [], "type": "validation_complete"} + + swarm = ValidatorSwarmAgent(mock_llm, mock_tools, mock_session, hunter_model="gpt-5.6-sol") + await swarm.run("validate", context={"hunter": {"findings": []}}) + + # Non-empty finding triggers swarm start and self-validation warning + result = await swarm.run("validate", context={ + "hunter": {"findings": [{"file_path": "auth.py", "category": "xss", "description": "test"}]} + }) + + # Check that warning trace event was recorded + warning_traces = [ + call.kwargs["content"] + for call in mock_session.add_trace.call_args_list + if call.kwargs.get("event_type") == "warning" + ] + assert len(warning_traces) >= 1 + assert warning_traces[0]["warning"] == "self_validation_detected" + + def test_swarm_sub_validator_model_independent(self, mock_tools, mock_session): + # Coordinator passes hunter_model gpt-5.6-sol and validator client with kimi-k2.5 + val_llm = MagicMock() + val_llm.model = "kimi-k2.5" + val_llm.provider = "openhack" + val_llm.prompt_cache_key = None + + with patch.object(settings, "validator_model_id", None): + swarm = ValidatorSwarmAgent(val_llm, mock_tools, mock_session, hunter_model="gpt-5.6-sol") + sub_llm = swarm._create_llm_for_sub_validator() + # Validator must use kimi-k2.5 and NOT invert back to gpt-5.6-sol + assert sub_llm.model == "kimi-k2.5" + + def test_swarm_sub_validator_model_automatic_when_hunter_model_unset(self, mock_tools, mock_session): + # Caller passes a default LLMClient without hunter_model (e.g. scripts/run_feature_hunt.py) + default_llm = MagicMock() + default_llm.model = "gpt-5.6-sol" + default_llm.provider = "openhack" + default_llm.prompt_cache_key = None + + with patch.object(settings, "validator_model_id", None), \ + patch.object(settings, "hunter_model_id", None): + swarm = ValidatorSwarmAgent(default_llm, mock_tools, mock_session, hunter_model=None) + sub_llm = swarm._create_llm_for_sub_validator() + # Must automatically select cross-model (kimi-k2.5) rather than self-validating with gpt-5.6-sol + assert sub_llm.model == "kimi-k2.5" + + def test_swarm_sub_validator_model_automatic_when_caller_passes_matching_generic_client(self, mock_tools, mock_session): + # Caller passes a generic LLMClient whose model matches hunter_model (e.g. scripts/run_feature_hunt.py) + default_llm = MagicMock() + default_llm.model = "gpt-5.6-sol" + default_llm.provider = "openhack" + default_llm.prompt_cache_key = None + + with patch.object(settings, "validator_model_id", None), \ + patch.object(settings, "hunter_model_id", None): + swarm = ValidatorSwarmAgent(default_llm, mock_tools, mock_session, hunter_model="gpt-5.6-sol") + sub_llm = swarm._create_llm_for_sub_validator() + # Must automatically select cross-model (kimi-k2.5) rather than self-validating with gpt-5.6-sol + assert sub_llm.model == "kimi-k2.5" + + @pytest.mark.asyncio + async def test_strict_majority_2_of_3_confirms_finding(self, mock_llm, mock_tools, mock_session): + # 3 votes: 2 confirmed, 1 false positive -> should be confirmed + with patch.object(settings, "validator_votes", 3), \ + patch.object(settings, "max_concurrent_validators", 5): + swarm = ValidatorSwarmAgent(mock_llm, mock_tools, mock_session) + + candidate_finding = { + "file_path": "server.py", + "category": "sqli", + "description": "SQL injection in query", + } + context = {"hunter": {"findings": [candidate_finding]}} + + run_count = 0 + async def mock_validator_run(task, ctx): + nonlocal run_count + run_count += 1 + if run_count in (1, 2): + return { + "validated_findings": [{ + "original_index": 0, + "status": "confirmed", + "confidence": "high", + "evidence": f"Confirmed in trial {run_count}", + "poc": "import requests", + "fix": "Use parameterized query", + }], + "false_positives": [], + "type": "validation_complete", + } + else: + return { + "validated_findings": [], + "false_positives": [{ + "original_index": 0, + "status": "false_positive", + "confidence": "medium", + "evidence": "Uncertain", + }], + "type": "validation_complete", + } + + with patch("openhack.agents.validator_swarm.ValidatorAgent.run", side_effect=mock_validator_run): + res = await swarm.run("validate", context=context) + + assert len(res["validated_findings"]) == 1 + assert res["validated_findings"][0]["status"] == "confirmed" + assert len(res["false_positives"]) == 0 + + @pytest.mark.asyncio + async def test_strict_majority_tie_is_rejected_not_confirmed(self, mock_llm, mock_tools, mock_session): + # 2 votes: 1 confirmed, 1 false positive -> tie must NOT confirm (strict majority required) + with patch.object(settings, "validator_votes", 2), \ + patch.object(settings, "max_concurrent_validators", 5): + swarm = ValidatorSwarmAgent(mock_llm, mock_tools, mock_session) + + candidate_finding = { + "file_path": "api.py", + "category": "csrf", + "description": "CSRF on state change", + } + context = {"hunter": {"findings": [candidate_finding]}} + + run_count = 0 + async def mock_validator_run(task, ctx): + nonlocal run_count + run_count += 1 + if run_count == 1: + return { + "validated_findings": [{ + "original_index": 0, + "status": "confirmed", + "evidence": "Trial 1 thinks confirmed", + }], + "false_positives": [], + "type": "validation_complete", + } + else: + return { + "validated_findings": [], + "false_positives": [{ + "original_index": 0, + "status": "false_positive", + "evidence": "Trial 2 found SameSite=Lax cookie", + }], + "type": "validation_complete", + } + + with patch("openhack.agents.validator_swarm.ValidatorAgent.run", side_effect=mock_validator_run): + res = await swarm.run("validate", context=context) + + # Tie must NOT be confirmed + assert len(res["validated_findings"]) == 0 + assert len(res["false_positives"]) == 1 + assert res["false_positives"][0]["status"] == "false_positive" + + @pytest.mark.asyncio + async def test_all_failed_votes_recorded_as_unreviewed_not_false_positive(self, mock_llm, mock_tools, mock_session): + # If all sub-validators fail due to an unhandled API/runtime crash, it's not a false positive + with patch.object(settings, "validator_votes", 3): + swarm = ValidatorSwarmAgent(mock_llm, mock_tools, mock_session) + context = {"hunter": {"findings": [{"file_path": "app.py", "category": "rce", "description": "eval"}]}} + + async def mock_failed_run(task, ctx): + return {"validated_findings": [], "false_positives": [], "type": "validation_failed"} + + with patch("openhack.agents.validator_swarm.ValidatorAgent.run", side_effect=mock_failed_run): + res = await swarm.run("validate", context=context) + + assert len(res["validated_findings"]) == 0 + assert len(res["false_positives"]) == 0 + assert len(res["unreviewed"]) == 1 + entry = res["unreviewed"][0] + assert entry["status"] == "unreviewed" + assert entry["confidence"] == "none" + assert "failed with execution errors" in entry["evidence"] + + @pytest.mark.asyncio + async def test_unfinished_review_with_partial_failures_recorded_as_unreviewed_not_rejection(self, mock_llm, mock_tools, mock_session): + # 3 votes requested: 1 confirmed, 2 failed execution errors -> cannot reach majority, but not rejected + with patch.object(settings, "validator_votes", 3): + swarm = ValidatorSwarmAgent(mock_llm, mock_tools, mock_session) + context = {"hunter": {"findings": [{"file_path": "app.py", "category": "sqli", "description": "query"}]}} + + run_count = 0 + async def mock_run(task, ctx): + nonlocal run_count + run_count += 1 + if run_count == 1: + return { + "validated_findings": [{"original_index": 0, "status": "confirmed", "evidence": "good"}], + "false_positives": [], + "type": "validation_complete", + } + return {"validated_findings": [], "false_positives": [], "type": "validation_failed"} + + with patch("openhack.agents.validator_swarm.ValidatorAgent.run", side_effect=mock_run): + res = await swarm.run("validate", context=context) + + # Must not be confirmed (only 1 of 2 needed votes) + assert len(res["validated_findings"]) == 0 + # Must NOT look like a rejection/false positive! + assert len(res["false_positives"]) == 0 + # Must be recorded separately as unreviewed/unfinished + assert len(res["unreviewed"]) == 1 + assert res["unreviewed"][0]["status"] == "unreviewed" + assert "unfinished or inconclusive" in res["unreviewed"][0]["evidence"] + + +class TestHardenedValidatorPromptChecks: + def test_prompt_includes_csrf_samesite_check(self): + assert "SameSite=Lax" in VALIDATOR_PROMPT + assert "SameSite=Strict" in VALIDATOR_PROMPT + assert "Set-Cookie" in VALIDATOR_PROMPT + + def test_prompt_includes_xxe_external_entity_check(self): + assert "XXE" in VALIDATOR_PROMPT + assert "EXTERNAL entities" in VALIDATOR_PROMPT + assert "xml2js" in VALIDATOR_PROMPT + + def test_prompt_includes_xss_sink_checks(self): + assert "dangerouslySetInnerHTML" in VALIDATOR_PROMPT + assert "RAW-HTML sink" in VALIDATOR_PROMPT + assert "auto-escaped" in VALIDATOR_PROMPT + assert "javascript:" in VALIDATOR_PROMPT + + def test_prompt_includes_framework_default_and_blank_rationale(self): + assert "Nuxt DevTools" in VALIDATOR_PROMPT or "Nuxt devtools" in VALIDATOR_PROMPT.lower() + assert "Blank / Vague Rationale" in VALIDATOR_PROMPT + + def test_prompt_defines_false_positive_strictly(self): + assert "MITIGATED, NOT PRESENT, NOT REACHABLE, or UNEXPLOITABLE" in VALIDATOR_PROMPT + + def test_validator_agent_system_prompt_builds_without_format_error(self): + # Verify that get_system_prompt() formats correctly without KeyError on literal braces + mock_llm = MagicMock() + mock_tools = MagicMock() + mock_session = MagicMock() + agent = ValidatorAgent(mock_llm, mock_tools, mock_session, original_finding_index=1) + context = { + "hunter": { + "findings": [{ + "file_path": "components/profile.tsx", + "category": "xss", + "severity": "high", + "line_number": 42, + "description": "User input in template", + "code_snippet": "
{userInput}
", + }] + }, + "project_context": {"framework": "nextjs"}, + } + prompt = agent.get_system_prompt(context) + assert "Candidate Finding 1 (UNVERIFIED HYPOTHESIS)" in prompt + assert "
{userInput}
" in prompt + assert "dangerouslySetInnerHTML" in prompt