v1.2.0 — Browser-driven real-traffic capture, per-program headers, two-agent workflow

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.2.0] - 2026-10-14
+
+### Added
+- **Browser-driven real-traffic capture** (`auto-scanner/integrations/browser_capture.js`
+  + `browser_capture.sh`) — drives a real headless Chromium via Playwright against
+  one authorized URL and records every request/response/WebSocket frame the page
+  actually makes, with real headers. This sees the requests a modern JS SPA
+  issues past client-gating walls (dynamic client-id, `Client-Version`,
+  anti-automation checks) that a static `curl` can't reproduce and that otherwise
+  return generic `4xx`/`410` walls. Scope-enforced before the browser launches;
+  degrades gracefully when node/Playwright are absent. Wired into the recon
+  pipeline as stage `[1.4d]` (skipped under `--passive-only`).
+- **Per-program custom-header injection** — the capture wrapper resolves a
+  mandatory research/identification header from `BB_EXTRA_HEADERS`,
+  `BB_RESEARCH_HEADER`, or auto-detected from the matching `programs/*.md`, and
+  injects it into every request the browser makes, so programs that require such
+  a header (e.g. `X-HackerOne-Research: <handle>`) can be scanned without
+  forfeiting eligibility.
+- **Two-agent research/verify workflow** (`docs/agent-workflow.md`) — codifies the
+  investigator + independent-reviewer pattern: every claim backed by something
+  executed and observed, severity never inflated beyond what was proven, and a
+  reviewer sign-off before any submission.
+
 ## [1.1.0] - 2026-10-06
 
 ### Added

auto-scanner/integrations/browser_capture.js

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+/*
+ * browser_capture.js — drive a real headless Chromium against ONE authorized URL
+ * and record every request/response/WebSocket frame the page makes, with real
+ * headers. This sees the requests a modern JS SPA actually issues — past the
+ * client-id / Client-Version / anti-automation gating that a static curl can't
+ * reproduce and that returns generic 4xx/410 walls.
+ *
+ * Scope enforcement and the mandatory per-program research header are handled by
+ * the browser_capture.sh wrapper BEFORE this runs. This script trusts the URL it
+ * is handed and simply injects whatever headers the wrapper resolved.
+ *
+ * Usage: browser_capture.js <url> <output-dir>
+ * Env:
+ *   BB_EXTRA_HEADERS   JSON object of headers to inject on every request
+ *                      (e.g. '{"X-HackerOne-Research":"dev101x"}')
+ *   BB_CAPTURE_MS      extra settle time after networkidle (default 6000)
+ *   BB_NAV_TIMEOUT_MS  navigation timeout (default 30000)
+ */
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+async function main() {
+  const url = process.argv[2];
+  const outDir = process.argv[3];
+  if (!url || !outDir) {
+    console.error("Usage: browser_capture.js <url> <output-dir>");
+    process.exit(2);
+  }
+
+  let chromium;
+  try {
+    ({ chromium } = require("playwright"));
+  } catch (e) {
+    console.error("playwright is not installed (npm install playwright). Skipping.");
+    process.exit(3);
+  }
+
+  let extraHeaders = {};
+  if (process.env.BB_EXTRA_HEADERS) {
+    try {
+      extraHeaders = JSON.parse(process.env.BB_EXTRA_HEADERS);
+    } catch (e) {
+      console.error("BB_EXTRA_HEADERS is not valid JSON; ignoring it.");
+    }
+  }
+  const settleMs = parseInt(process.env.BB_CAPTURE_MS || "6000", 10);
+  const navTimeout = parseInt(process.env.BB_NAV_TIMEOUT_MS || "30000", 10);
+
+  fs.mkdirSync(outDir, { recursive: true });
+
+  const requests = [];
+  const wsEvents = [];
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext({
+    // A real desktop UA so the SPA serves its normal code path, not a bot page.
+    userAgent:
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
+      "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    ignoreHTTPSErrors: false,
+  });
+  if (Object.keys(extraHeaders).length) {
+    await context.setExtraHTTPHeaders(extraHeaders);
+  }
+
+  const page = await context.newPage();
+
+  page.on("request", (req) => {
+    requests.push({
+      phase: "request",
+      method: req.method(),
+      url: req.url(),
+      resourceType: req.resourceType(),
+      headers: req.headers(),
+      postData: safePostData(req),
+      ts: Date.now(),
+    });
+  });
+
+  page.on("response", async (res) => {
+    let status = null;
+    try {
+      status = res.status();
+    } catch (e) {
+      /* response may be gone */
+    }
+    requests.push({
+      phase: "response",
+      url: res.url(),
+      status,
+      headers: safeHeaders(res),
+      ts: Date.now(),
+    });
+  });
+
+  page.on("websocket", (ws) => {
+    const wsUrl = ws.url();
+    wsEvents.push({ event: "open", url: wsUrl, ts: Date.now() });
+    ws.on("framesent", (data) =>
+      wsEvents.push({ event: "sent", url: wsUrl, payload: truncate(data.payload), ts: Date.now() })
+    );
+    ws.on("framereceived", (data) =>
+      wsEvents.push({ event: "received", url: wsUrl, payload: truncate(data.payload), ts: Date.now() })
+    );
+    ws.on("close", () => wsEvents.push({ event: "close", url: wsUrl, ts: Date.now() }));
+  });
+
+  let navError = null;
+  try {
+    await page.goto(url, { waitUntil: "domcontentloaded", timeout: navTimeout });
+    await page.waitForLoadState("networkidle", { timeout: navTimeout }).catch(() => {});
+    await page.waitForTimeout(settleMs);
+  } catch (e) {
+    navError = String(e && e.message ? e.message : e);
+  }
+
+  await browser.close();
+
+  // Distinct host:path endpoints the page actually called (the useful recon artifact).
+  const endpoints = {};
+  for (const r of requests) {
+    if (r.phase !== "request") continue;
+    try {
+      const u = new URL(r.url);
+      const key = `${r.method} ${u.host}${u.pathname}`;
+      if (!endpoints[key]) endpoints[key] = { count: 0, resourceType: r.resourceType, sampleUrl: r.url };
+      endpoints[key].count++;
+    } catch (e) {
+      /* skip non-URL */
+    }
+  }
+
+  const result = {
+    target: url,
+    capturedAt: new Date().toISOString(),
+    navError,
+    injectedHeaders: Object.keys(extraHeaders),
+    requestCount: requests.filter((r) => r.phase === "request").length,
+    websocketCount: wsEvents.filter((w) => w.event === "open").length,
+    endpoints,
+  };
+
+  fs.writeFileSync(path.join(outDir, "browser_capture.json"), JSON.stringify(result, null, 2));
+  fs.writeFileSync(path.join(outDir, "browser_capture_raw.json"), JSON.stringify({ requests, wsEvents }, null, 2));
+
+  // Flat endpoint list for grep-ability / downstream tooling.
+  fs.writeFileSync(
+    path.join(outDir, "browser_endpoints.txt"),
+    Object.keys(endpoints).sort().join("\n") + "\n"
+  );
+
+  console.error(
+    `captured ${result.requestCount} requests, ${Object.keys(endpoints).length} distinct endpoints, ${result.websocketCount} websocket(s)` +
+      (navError ? ` (nav warning: ${navError})` : "")
+  );
+}
+
+function safePostData(req) {
+  try {
+    const d = req.postData();
+    return d ? truncate(d) : null;
+  } catch (e) {
+    return null;
+  }
+}
+
+function safeHeaders(res) {
+  try {
+    return res.headers();
+  } catch (e) {
+    return {};
+  }
+}
+
+function truncate(s) {
+  s = typeof s === "string" ? s : String(s);
+  return s.length > 4096 ? s.slice(0, 4096) + "...[truncated]" : s;
+}
+
+main().catch((e) => {
+  console.error("browser_capture fatal:", e && e.message ? e.message : e);
+  process.exit(1);
+});

auto-scanner/integrations/browser_capture.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Browser-driven real-traffic capture for the scope-enforced bug bounty workflow.
+#
+# Drives a real headless Chromium (via Playwright) against ONE authorized URL and
+# records every request/response/WebSocket frame the page makes — the requests a
+# modern JS SPA actually issues, past client-gating walls a static curl can't pass.
+#
+# Scope is enforced here before the browser ever launches. The mandatory
+# per-program research/identification header is resolved here and injected into
+# every request the browser makes, so automated capture stays policy-compliant.
+#
+# Usage: browser_capture.sh <authorized-url> [output-directory]
+# Env:
+#   BB_EXTRA_HEADERS   JSON object of headers to inject (overrides auto-detection)
+#   BB_RESEARCH_HEADER explicit "Name: value" header (e.g. "X-HackerOne-Research: dev101x")
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=../lib/common.sh
+source "$SCRIPT_DIR/../lib/common.sh"
+
+usage() {
+    echo "Usage: $0 <authorized-url> [output-directory]" >&2
+}
+
+TARGET_URL="${1:-}"
+if [ -z "$TARGET_URL" ]; then
+    usage
+    exit 1
+fi
+
+# Scope gate: refuses any URL whose host is not authorized in programs/.
+require_scope "$TARGET_URL" || exit 1
+
+DOMAIN="$(normalize_target "$TARGET_URL")" || exit 1
+OUTPUT_DIR="${2:-$REPO_ROOT/bugbounty/reports/$DOMAIN/recon}"
+mkdir -p "$OUTPUT_DIR"
+
+if ! check_tool node; then
+    log_warning "node is not installed; skipping browser capture."
+    exit 0
+fi
+
+# Resolve the headers to inject. Priority:
+#   1. BB_EXTRA_HEADERS (explicit JSON, used as-is)
+#   2. BB_RESEARCH_HEADER ("Name: value" -> one-entry JSON)
+#   3. auto-detect from the matching programs/*.md "Mandatory header" line
+if [ -z "${BB_EXTRA_HEADERS:-}" ]; then
+    if [ -n "${BB_RESEARCH_HEADER:-}" ]; then
+        hname="${BB_RESEARCH_HEADER%%:*}"
+        hval="${BB_RESEARCH_HEADER#*:}"
+        hname="$(printf '%s' "$hname" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
+        hval="$(printf '%s' "$hval" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
+        export BB_EXTRA_HEADERS
+        BB_EXTRA_HEADERS="$(printf '{"%s":"%s"}' "$hname" "$hval")"
+    else
+        # Pull a `X-...-Research: <value>` style header out of any program file
+        # whose scope covers this target. Narrow + best-effort: only matches the
+        # documented backtick form `Header: value`.
+        detected="$(grep -rohiE '`[A-Za-z-]*[Rr]esearch[A-Za-z-]*:[[:space:]]*[A-Za-z0-9_.-]+`' "$PROGRAMS_DIR" 2>/dev/null | head -n1 | tr -d '`')"
+        if [ -n "$detected" ]; then
+            hname="${detected%%:*}"
+            hval="${detected#*:}"
+            hname="$(printf '%s' "$hname" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
+            hval="$(printf '%s' "$hval" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
+            export BB_EXTRA_HEADERS
+            BB_EXTRA_HEADERS="$(printf '{"%s":"%s"}' "$hname" "$hval")"
+            log_info "Auto-detected research header from programs/: $hname"
+        fi
+    fi
+fi
+
+if [ -n "${BB_EXTRA_HEADERS:-}" ]; then
+    log_info "Injecting headers into every captured request."
+else
+    log_warning "No research/identification header set. If this program requires one, set BB_RESEARCH_HEADER before capturing."
+fi
+
+log_info "Browser capture starting against: $TARGET_URL"
+node "$SCRIPT_DIR/browser_capture.js" "$TARGET_URL" "$OUTPUT_DIR"
+rc=$?
+
+case "$rc" in
+    0) log_info "Capture written to $OUTPUT_DIR/browser_capture.json (+ _raw.json, browser_endpoints.txt)" ;;
+    3) log_warning "Playwright not installed. Run: npm install playwright && npx playwright install chromium" ;;
+    *) log_warning "Browser capture exited with code $rc" ;;
+esac
+exit "$rc"

bugbounty/bugbounty-hunter.sh

@@ -11,7 +11,7 @@ RED='\033[0;31m'
 YELLOW='\033[1;33m'
 NC='\033[0m'
 
-BB_VERSION="1.1.0"
+BB_VERSION="1.2.0"
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 BUGBOUNTY_DIR="$SCRIPT_DIR"
@@ -340,6 +340,16 @@ reconnaissance() {
             code=$(curl -s -o /dev/null -w "%{http_code}" "https://${TARGET}${sp}" 2>/dev/null || echo "000")
             [ "$code" = "200" ] && echo "https://${TARGET}${sp}  ($code)" >> "$OUTPUT_DIR/api_specs.txt"
         done
+
+        echo -e "${YELLOW}[1.4d] Browser-driven real-traffic capture${NC}"
+        # Drive a real headless Chromium so we see the requests the SPA actually
+        # makes (real headers, XHR/fetch/WS) past client-gating walls a static
+        # curl can't pass. Scope-enforced + mandatory research header injected by
+        # the wrapper. Optional: skipped cleanly when node/playwright absent.
+        if [ -x "$BUGBOUNTY_DIR/../auto-scanner/integrations/browser_capture.sh" ]; then
+            "$BUGBOUNTY_DIR/../auto-scanner/integrations/browser_capture.sh" \
+                "https://$TARGET" "$OUTPUT_DIR" || true
+        fi
     fi
 
     echo -e "${YELLOW}[1.5] Parameter discovery${NC}"