feat(agents): cross-model validation and strict majority consensus (P0)

openhack/agents/coordinator.py

@@ -39,7 +39,7 @@
 from openhack.framework_detection import detect_frameworks
 from openhack.quality import run_quality_gates
 # Static validator removed — line number correction in hunter, semantic validation by LLM
-from openhack.config import settings
+from openhack.config import settings, resolve_validator_model, check_self_validation
 
 logger = logging.getLogger(__name__)
 
@@ -72,7 +72,15 @@ def get_system_prompt(self, context: dict) -> str:
 
     def _create_llm_for_agent(self, agent_type: str) -> LLMClient:
         model_override = getattr(settings, f"{agent_type}_model_id", None)
-        model = model_override or self.llm.model
+        if agent_type == "validator":
+            hunter_model = getattr(settings, "hunter_model_id", None) or self.llm.model
+            model = resolve_validator_model(
+                hunter_model=hunter_model,
+                requested_validator_model=model_override,
+                provider=self.llm.provider,
+            )
+        else:
+            model = model_override or self.llm.model
         return LLMClient(model=model, temperature=0.0, max_tokens=8192, provider=self.llm.provider, prompt_cache_key=self.llm.prompt_cache_key)
 
     @staticmethod
@@ -790,7 +798,10 @@ async def run_pass2(task_text, batch_idx):
             if potential_findings:
                 self.session.add_trace(agent="coordinator", event_type="step_start", content="Step 3: Validation (swarm)")
                 validator_llm = self._create_llm_for_agent("validator")
-                validator_swarm = ValidatorSwarmAgent(validator_llm, self.tools, self.session)
+                hunter_model = getattr(settings, "hunter_model_id", None) or self.llm.model
+                validator_swarm = ValidatorSwarmAgent(
+                    validator_llm, self.tools, self.session, hunter_model=hunter_model
+                )
                 validator_result = await validator_swarm.run(
                     "Validate each potential vulnerability.", context=self.context)
                 self.context["validator"] = validator_result

openhack/agents/validator.py

@@ -68,16 +68,20 @@ def get_system_prompt(self, context: dict) -> str:
         findings_text = ""
         for i, f in enumerate(findings, 1):
             findings_text += f"""
-### Finding {i}
+### Candidate Finding {i} (UNVERIFIED HYPOTHESIS)
 - **Category**: {f.get('category', 'Unknown')}
-- **Severity**: {f.get('severity', 'Unknown')}
+- **Reported Severity**: {f.get('severity', 'Unknown')}
 - **File**: {f.get('file_path', 'Unknown')}
 - **Line**: {f.get('line_number', 'Unknown')}
-- **Description**: {f.get('description', 'No description')}
-- **Code**:
+- **Description / Claim**: {f.get('description', 'No description')}
+- **Code Snippet**:
 ```
 {f.get('code_snippet', 'No code snippet')}
 ```
+> Note: Treat this candidate finding as an unverified hypothesis from the hunter swarm.
+> Independently audit the surrounding code, data flow, and mitigations. If the issue is mitigated,
+> uses framework auto-escaping, lacks external entity resolution, relies on SameSite cookies,
+> or is a framework dev default, mark it as false_positive.
 """
 
         base_prompt = VALIDATOR_PROMPT.format(

openhack/agents/validator_swarm.py

@@ -10,7 +10,7 @@
 from .llm import LLMClient
 from .session import Session
 from openhack.tools.registry import ToolRegistry
-from openhack.config import settings
+from openhack.config import settings, resolve_validator_model, check_self_validation
 
 logger = logging.getLogger(__name__)
 
@@ -19,18 +19,48 @@ class ValidatorSwarmAgent:
     name = "validator_swarm"
     description = "Validator swarm coordinator"
 
-    def __init__(self, llm: LLMClient, tools: ToolRegistry, session: Session):
+    def __init__(
+        self,
+        llm: LLMClient,
+        tools: ToolRegistry,
+        session: Session,
+        hunter_model: Optional[str] = None,
+    ):
         self.llm = llm
         self.tools = tools
         self.session = session
+        self.hunter_model = hunter_model
         self.total_cost: float = 0.0
         self.total_tokens: int = 0
         self.total_input_tokens: int = 0
         self.total_output_tokens: int = 0
 
+    def _get_models(self) -> tuple[str, str]:
+        hunter_model = self.hunter_model or getattr(settings, "hunter_model_id", None) or self.llm.model
+        if settings.validator_model_id:
+            validator_model = settings.validator_model_id
+        elif self.llm.model and self.llm.model != hunter_model:
+            # Caller passed an already-resolved validator client distinct from the hunter model
+            validator_model = self.llm.model
+        else:
+            # Caller passed a generic/default client without resolving an independent validator model;
+            # preserve automatic cross-model selection so hunter does not review its own findings
+            validator_model = resolve_validator_model(
+                hunter_model=hunter_model,
+                requested_validator_model=None,
+                provider=self.llm.provider,
+            )
+        return hunter_model, validator_model
+
     def _create_llm_for_sub_validator(self) -> LLMClient:
-        model = settings.validator_model_id or self.llm.model
-        return LLMClient(model=model, temperature=0.0, max_tokens=8192, provider=self.llm.provider, prompt_cache_key=self.llm.prompt_cache_key)
+        _, validator_model = self._get_models()
+        return LLMClient(
+            model=validator_model,
+            temperature=0.0,
+            max_tokens=8192,
+            provider=self.llm.provider,
+            prompt_cache_key=self.llm.prompt_cache_key,
+        )
 
     def _build_sub_context(self, finding: dict, full_context: dict) -> dict:
         return {
@@ -44,32 +74,69 @@ async def run(self, task: str, context: Optional[dict] = None) -> dict:
         findings = context.get("hunter", {}).get("findings", [])
 
         if not findings:
-            return {"raw_output": "No findings to validate", "validated_findings": [], "false_positives": [], "type": "validation_complete"}
+            return {
+                "raw_output": "No findings to validate",
+                "validated_findings": [],
+                "false_positives": [],
+                "unreviewed": [],
+                "type": "validation_complete",
+            }
+
+        hunter_model, validator_model = self._get_models()
+
+        # Startup check: warn if validator uses the same model as hunter (self-validation)
+        self_val_warning = check_self_validation(hunter_model, validator_model)
+        if self_val_warning:
+            logger.warning(self_val_warning)
+            self.session.add_trace(
+                agent=self.name,
+                event_type="warning",
+                content={
+                    "warning": "self_validation_detected",
+                    "hunter_model": hunter_model,
+                    "validator_model": validator_model,
+                    "message": self_val_warning,
+                },
+            )
 
-        self.session.add_trace(agent=self.name, event_type="swarm_start", content={"findings_count": len(findings)})
+        votes = max(1, getattr(settings, "validator_votes", 1))
+        self.session.add_trace(
+            agent=self.name,
+            event_type="swarm_start",
+            content={
+                "findings_count": len(findings),
+                "hunter_model": hunter_model,
+                "validator_model": validator_model,
+                "validator_votes": votes,
+            },
+        )
 
-        sub_validators: list[tuple[int, ValidatorAgent, dict]] = []
+        sub_validators: list[tuple[int, int, ValidatorAgent, dict]] = []
         for idx, finding in enumerate(findings):
-            llm = self._create_llm_for_sub_validator()
-            validator = ValidatorAgent(llm, self.tools, self.session, original_finding_index=idx)
             sub_context = self._build_sub_context(finding, context)
-            sub_validators.append((idx, validator, sub_context))
+            for vote_idx in range(votes):
+                llm = self._create_llm_for_sub_validator()
+                validator = ValidatorAgent(llm, self.tools, self.session, original_finding_index=idx)
+                sub_validators.append((idx, vote_idx, validator, sub_context))
 
         semaphore = asyncio.Semaphore(settings.max_concurrent_validators)
 
-        async def run_sub_validator(finding_idx, validator, sub_context):
+        async def run_sub_validator(finding_idx, vote_idx, validator, sub_context):
             async with semaphore:
                 try:
-                    sub_task = "Validate this potential vulnerability. Confirm whether it is real, generate a PoC, and suggest a fix."
+                    sub_task = (
+                        "Validate this potential vulnerability. Confirm whether it is real, "
+                        "generate a PoC, and suggest a fix."
+                    )
                     result = await validator.run(sub_task, sub_context)
-                    return finding_idx, result
+                    return finding_idx, vote_idx, result
                 except Exception as e:
-                    logger.error(f"Sub-validator for finding {finding_idx} failed: {e}")
-                    return finding_idx, {"validated_findings": [], "false_positives": [], "type": "validation_failed"}
+                    logger.error(f"Sub-validator for finding {finding_idx} (vote {vote_idx}) failed: {e}")
+                    return finding_idx, vote_idx, {"validated_findings": [], "false_positives": [], "type": "validation_failed"}
 
         tasks = [
-            asyncio.create_task(run_sub_validator(idx, validator, sub_ctx))
-            for idx, validator, sub_ctx in sub_validators
+            asyncio.create_task(run_sub_validator(idx, vote_idx, validator, sub_ctx))
+            for idx, vote_idx, validator, sub_ctx in sub_validators
         ]
         try:
             results = await asyncio.gather(*tasks)
@@ -79,28 +146,125 @@ async def run_sub_validator(finding_idx, validator, sub_context):
             await asyncio.gather(*tasks, return_exceptions=True)
             raise
 
+        results_by_finding: dict[int, list[dict]] = {idx: [] for idx in range(len(findings))}
+        for finding_idx, vote_idx, result in results:
+            results_by_finding[finding_idx].append(result)
+
         all_validated: list[dict] = []
         all_false_positives: list[dict] = []
+        all_unreviewed: list[dict] = []
+        threshold = (votes // 2) + 1
+
+        for finding_idx, run_results in results_by_finding.items():
+            confirmed_runs = [
+                r for r in run_results if r.get("validated_findings")
+            ]
+            confirmed_count = len(confirmed_runs)
+            rejection_runs = [
+                r for r in run_results if r.get("false_positives")
+            ]
+            failed_runs = [
+                r for r in run_results if r.get("type") == "validation_failed"
+            ]
 
-        for finding_idx, result in results:
-            all_validated.extend(result.get("validated_findings", []))
-            all_false_positives.extend(result.get("false_positives", []))
+            # Strict majority required (>= threshold). Ties are NOT confirmed.
+            if confirmed_count >= threshold and confirmed_runs:
+                best_run = max(
+                    confirmed_runs,
+                    key=lambda r: len(r["validated_findings"][0].get("evidence", ""))
+                    if r["validated_findings"]
+                    else 0,
+                )
+                all_validated.extend(best_run.get("validated_findings", []))
+                self.session.add_trace(
+                    agent=self.name,
+                    event_type="validation_consensus",
+                    content={
+                        "finding_index": finding_idx,
+                        "status": "confirmed",
+                        "confirmed_votes": confirmed_count,
+                        "total_votes": votes,
+                        "threshold": threshold,
+                    },
+                )
+            elif (len(failed_runs) > 0 and len(rejection_runs) < threshold) or not run_results:
+                # Review is unfinished or inconclusive because one or more votes failed execution
+                # (API error, prompt error, or timeout) without achieving a definitive rejection.
+                # Record unfinished/failed review separately from an actual rejection.
+                unreviewed_entry = {
+                    "original_index": finding_idx,
+                    "status": "unreviewed",
+                    "confidence": "none",
+                    "evidence": (
+                        f"Validation review was unfinished or inconclusive ({confirmed_count} confirmed, "
+                        f"{len(rejection_runs)} rejected, {len(failed_runs)} failed with execution errors across {votes} attempts)."
+                    ),
+                }
+                all_unreviewed.append(unreviewed_entry)
+                self.session.add_trace(
+                    agent=self.name,
+                    event_type="validation_consensus",
+                    content={
+                        "finding_index": finding_idx,
+                        "status": "unreviewed",
+                        "confirmed_votes": confirmed_count,
+                        "rejected_votes": len(rejection_runs),
+                        "failed_votes": len(failed_runs),
+                        "total_votes": votes,
+                        "threshold": threshold,
+                        "review_failed": True,
+                    },
+                )
+            else:
+                fp_entry = None
+                for r in run_results:
+                    if r.get("false_positives"):
+                        fp_entry = r["false_positives"][0]
+                        break
+                if not fp_entry:
+                    fp_entry = {
+                        "original_index": finding_idx,
+                        "status": "false_positive",
+                        "confidence": "medium",
+                        "evidence": f"Failed strict majority validation ({confirmed_count}/{votes} confirmed votes, required >={threshold})",
+                    }
+                all_false_positives.append(fp_entry)
+                self.session.add_trace(
+                    agent=self.name,
+                    event_type="validation_consensus",
+                    content={
+                        "finding_index": finding_idx,
+                        "status": fp_entry.get("status", "false_positive"),
+                        "confirmed_votes": confirmed_count,
+                        "total_votes": votes,
+                        "threshold": threshold,
+                        "review_failed": False,
+                    },
+                )
 
-        for _, validator, _ in sub_validators:
+        for _, _, validator, _ in sub_validators:
             self.total_cost += validator.llm.total_cost
             self.total_tokens += validator.llm.total_tokens
             self.total_input_tokens += validator.llm.total_input_tokens
             self.total_output_tokens += validator.llm.total_output_tokens
 
         self.session.add_trace(
-            agent=self.name, event_type="swarm_complete",
-            content={"total_confirmed": len(all_validated), "total_false_positives": len(all_false_positives),
-                     "total_cost": self.total_cost, "total_tokens": self.total_tokens},
+            agent=self.name,
+            event_type="swarm_complete",
+            content={
+                "total_confirmed": len(all_validated),
+                "total_false_positives": len(all_false_positives),
+                "total_unreviewed": len(all_unreviewed),
+                "total_cost": self.total_cost,
+                "total_tokens": self.total_tokens,
+                "validator_votes": votes,
+            },
         )
 
         return {
-            "raw_output": f"Validated {len(findings)} findings: {len(all_validated)} confirmed, {len(all_false_positives)} false positives",
+            "raw_output": f"Validated {len(findings)} findings: {len(all_validated)} confirmed, {len(all_false_positives)} false positives, {len(all_unreviewed)} unreviewed",
             "validated_findings": all_validated,
             "false_positives": all_false_positives,
+            "unreviewed": all_unreviewed,
             "type": "validation_complete",
         }

openhack/config.py

@@ -140,6 +140,9 @@ class Settings(BaseSettings):
     validator_model_id: Optional[str] = None
     browser_verifier_model_id: Optional[str] = None
 
+    # Number of validation votes per finding. 1 = single pass. Set to 3 for strict >=2-of-3 majority.
+    validator_votes: int = 1
+
     max_concurrent_hunters: int = 3
     max_concurrent_validators: int = 5
 
@@ -257,3 +260,46 @@ def reload_settings() -> None:
     fresh = _build_settings()
     for field_name in Settings.model_fields:
         setattr(settings, field_name, getattr(fresh, field_name))
+
+
+# Cross-model validation defaults: mapping hunter model to an independent validator model
+DEFAULT_CROSS_MODELS = {
+    "gpt-5.6-sol": "kimi-k2.5",
+    "gpt-5-mini": "kimi-k2.5",
+    "glm-5.2": "kimi-k2.5",
+    "glm-5.3-flash": "kimi-k2.5",
+    "kimi-k2.5": "gpt-5.6-sol",
+    "claude-sonnet-5": "gpt-5.6-sol",
+    "meta-llama/Llama-3.3-70B-Instruct": "gpt-5.6-sol",
+}
+
+
+def resolve_validator_model(
+    hunter_model: str,
+    requested_validator_model: Optional[str] = None,
+    provider: str = "openhack",
+) -> str:
+    """Resolve validator model, preferring cross-model validation over self-validation."""
+    if requested_validator_model:
+        return requested_validator_model
+    if provider == "openhack":
+        # Hosted OpenHack has access to cross-model targets (e.g. K2.5 vs GPT-5.6 / GLM)
+        return DEFAULT_CROSS_MODELS.get(
+            hunter_model,
+            "kimi-k2.5" if hunter_model != "kimi-k2.5" else "gpt-5.6-sol",
+        )
+    return hunter_model
+
+
+def check_self_validation(hunter_model: str, validator_model: str) -> Optional[str]:
+    """Return warning message if validator model is identical to hunter model."""
+    if (
+        hunter_model
+        and validator_model
+        and hunter_model.strip().lower() == validator_model.strip().lower()
+    ):
+        return (
+            f"Self-validation detected: validator using same model as hunter ({validator_model}). "
+            "Cross-model validation recommended to prevent precision inflation."
+        )
+    return None