diff --git a/.github/workflows/site-numbers.yml b/.github/workflows/site-numbers.yml new file mode 100644 index 0000000..138e66c --- /dev/null +++ b/.github/workflows/site-numbers.yml @@ -0,0 +1,68 @@ +# Site numbers come from the claims register in bawbel/.github (scripts/sync_site_numbers.py). +# On a pull request: fail if the stat block is stale or a count is typed outside it. +# Weekly and on demand: regenerate the block and open a pull request against develop. +name: site numbers + +on: + pull_request: + branches: [develop, main] + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: python -m pip install -r scripts/requirements.txt + - run: python -m pytest tests -q + - run: python scripts/sync_site_numbers.py --check + + update: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + ref: develop + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - run: python -m pip install -r scripts/requirements.txt + - run: python scripts/sync_site_numbers.py + - name: Open a pull request when the numbers changed + env: + GH_TOKEN: ${{ github.token }} + run: | + if git diff --quiet; then + echo "Stat block already current." + exit 0 + fi + if [ -n "$(gh pr list --base develop --state open --search 'head:chore/site-numbers-' --json number --jq '.[].number')" ]; then + echo "A site-numbers pull request is already open." + exit 0 + fi + branch="chore/site-numbers-$(date -u +%Y%m%d)" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$branch" + git commit -s -am "chore: sync site numbers from the claims register" + git push origin "$branch" + gh pr create --base develop --head "$branch" \ + --title "chore: sync site numbers from the claims register" \ + --body "Generated by scripts/sync_site_numbers.py from claims/register.yaml in bawbel/.github. Review the new values against that register before merging." diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6c56ff1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +.pytest_cache/ diff --git a/README.md b/README.md index 487fb81..b40cec6 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ before they reach production. Free forever. Apache 2.0. [](https://pypi.org/project/bawbel-scanner/) [](https://pepy.tech/project/bawbel-scanner) -[](https://github.com/bawbel/ave) +[](https://github.com/bawbel/ave) [](https://github.com/bawbel/scanner/blob/main/LICENSE) @@ -23,7 +23,7 @@ before they reach production. Free forever. Apache 2.0. | Repo | What it is | |---|---| | [bawbel/scanner](https://github.com/bawbel/scanner) | CLI scanner for agentic AI components -- SKILL.md files, MCP servers, system prompts. Confidence and evidence metadata on every finding. | -| [bawbel/ave](https://github.com/bawbel/ave) | AVE -- the behavioral classification standard for agentic AI components. 51 published records, scored with OWASP AIVSS v0.8, mapped to OWASP MCP Top 10 and MITRE ATLAS. | +| [bawbel/ave](https://github.com/bawbel/ave) | AVE -- the behavioral classification standard for agentic AI components. Published records scored with OWASP AIVSS v0.8, mapped to OWASP MCP Top 10 and MITRE ATLAS. | | [bawbel/piranha-api](https://github.com/bawbel/piranha-api) | Free, no-auth REST API serving the full AVE record set and threat intelligence. | | [bawbel/integrations](https://github.com/bawbel/integrations) | GitHub Action, VS Code extension, and pre-commit hooks. | @@ -46,6 +46,17 @@ Your CI pipeline scans dependencies for known package vulnerabilities. It does n --- +### Site numbers + +Every count on bawbel.io comes from the claims register in +[bawbel/.github](https://github.com/bawbel/.github/blob/main/claims/register.yaml), which that +repository's verifier checks against the code. `scripts/sync_site_numbers.py` writes the values +into the stat block in `index.html`. A pull request fails when the block is stale or a count is +typed anywhere else, and a weekly workflow opens a pull request when the register changes. To +change a number, change the register. + +--- + ### Links [bawbel.io](https://bawbel.io) ยท diff --git a/docs/index.html b/docs/index.html index baadd61..bdcaeba 100644 --- a/docs/index.html +++ b/docs/index.html @@ -434,12 +434,12 @@
Pattern, YARA, and Semgrep rules mapped to all 51 AVE records. Every finding now carries confidence and evidence metadata.
+Pattern, YARA, and Semgrep rules, each tied to the AVE record it detects. Every finding now carries confidence and evidence metadata.
Pattern, YARA, Semgrep, LLM semantic analysis, Magika file-type verification, Docker behavioral sandbox.
Full registry of all 51 records: ave.bawbel.io/registry.html
+Full registry of published records: aveproject.org/registry.html
BAWBEL_SANDBOX_ENABLED=trueIndividual findings that are each only MEDIUM severity can combine into a CRITICAL attack chain. 12 built-in chains detect exactly this -- for example, a credential-read finding plus a data-exfiltration finding in the same component becomes a single Credential Exfiltration Chain at CRITICAL 9.8, rather than two separate MEDIUM findings a reviewer might triage as low priority individually.
+Individual findings that are each only MEDIUM severity can combine into a CRITICAL attack chain. Built-in chains detect exactly this -- for example, a credential-read finding plus a data-exfiltration finding in the same component becomes a single Credential Exfiltration Chain at CRITICAL 9.8, rather than two separate MEDIUM findings a reviewer might triage as low priority individually.
$ bawbel scan ./skills/ --recursive
CRIT AVE-2026-00003 line 4 AIVSS 6.8 confidence 0.75
@@ -857,7 +857,7 @@ Toxic flows
Chain confidence is the minimum confidence across all contributing findings -- the weakest link in the chain, not the average. This is deliberate: a chain is only as trustworthy as its least certain component.
- Use bawbel chain to filter specifically to delegation-chain findings (AVE-2026-00048), or run a full bawbel scan to see all 12 toxic flow chains evaluated together.
+ Use bawbel chain to filter specifically to delegation-chain findings (AVE-2026-00048), or run a full bawbel scan to see every toxic flow chain evaluated together.
12 built-in attack chains. Credential-read plus data-exfil becomes CRITICAL 9.8 automatically. Combined AIVSS score and minimum confidence across the full chain.
+Built-in attack chains. Credential-read plus data-exfil becomes CRITICAL 9.8 automatically. Combined AIVSS score and minimum confidence across the full chain.
Pattern (40 rules), YARA (39), Semgrep (41), LLM semantic, Magika content-type, Docker sandbox. 121 rules total across 51 AVE records.
+Pattern, YARA, and Semgrep rules, LLM semantic analysis, Magika content-type checks, and a Docker sandbox stage that is off by default. Each rule is tied to the AVE record it detects.
owasp_mcp is required on every one of the 51 AVE records, MCP01 through MCP10. Every finding your scan produces lands in a category your security team already tracks.
owasp_mcp is required on every AVE record, MCP01 through MCP10. Every finding your scan produces lands in a category your security team already tracks.
| Feature | -Bawbel | -Snyk agent-scan | -ClawGuard | -Cisco DefenseClaw | +Feature | +Bawbel | +NVIDIA SkillSpector | +Snyk agent-scan | +Cisco DefenseClaw |
|---|---|---|---|---|---|---|---|---|---|
| Executes MCP servers during scan | Never | Yes | No | Sandboxed | |||||
| Open behavioral classification standard | Yes (51 records) | No | No | No | |||||
| OWASP AIVSS v0.8 scores | Yes | No | No | No | |||||
| Confidence and evidence metadata | Yes | No | No | No | |||||
| Toxic flow detection | Yes (12 chains) | No | No | No | |||||
| Justified suppression with expiry | Yes | No | No | No | |||||
| Conformance grading | Yes (A+ to F) | No | No | No | |||||
| License | Apache 2.0 | Apache 2.0 | MIT | Proprietary | |||||
| Starts MCP servers during a scan | Never | Never | Yes (stdio servers) | ? | |||||
| Runs scanned skill content | Only in the optional Docker sandbox stage, off by default | Never | ? | ? | |||||
| Open vulnerability database | Yes (AVE records, public API) | No (internal rule IDs) | No (internal rule IDs) | ? | |||||
| OWASP AIVSS v0.8 scores | Yes | No (custom 0-100 risk score) | No | ? | |||||
| Compound attack chain detection | Yes (AVE capability chains) | Yes (TT1-TT5 taint flows + AST8 chain) | Yes (chain count not published) | ? | |||||
| MCP spec conformance grading (A+ to F) | Yes | No | No | ? | |||||
| Git-committed pin / rug pull detection | Yes | No | No | ? | |||||
| Suppression with expiry and audit trail | Yes (reason, reviewer, expiry) | Baseline (glob and fingerprint, no expiry) | Baseline (ignore flags, no expiry) | ? | |||||
| Per-scan LLM usage and cost telemetry | No (tracked in #79) | Yes (metadata.inference_usage) | No | ? | |||||
| SARIF 2.1.0 output | Yes | Yes | No | ? | |||||
| License | Apache 2.0 | Apache 2.0 | Apache 2.0 | Apache 2.0 |
? means not verifiable from each project's public README at the time of writing (2026-09-12). The same table, with its sources, is in the scanner README.
Free, no auth, Apache 2.0. Every scanner finding includes a piranha_url linking to the full record with IOCs and remediation.
", re.S | re.I)
+
+
+class RegisterError(Exception):
+ """The register could not be read, or lacks a value the site needs."""
+
+
+def read_register(source: str) -> dict:
+ """The parsed register from a file path or an https URL."""
+ if os.path.isfile(source):
+ text = Path(source).read_text(encoding="utf-8")
+ else:
+ parts = urlsplit(source)
+ if parts.scheme != "https" or not parts.netloc:
+ raise RegisterError(f"not a file or an https URL: {source}")
+ req = urllib.request.Request(source, headers={"User-Agent": "bawbel-site-numbers"})
+ try:
+ # B310: the scheme is checked above, so only https reaches urlopen.
+ with urllib.request.urlopen(req, timeout=FETCH_TIMEOUT_SEC) as resp: # nosec B310
+ body = resp.read(MAX_REGISTER_BYTES + 1)
+ except OSError as exc: # URLError, HTTPError, and timeouts are all OSError
+ raise RegisterError(f"{source}: {exc}") from exc
+ if len(body) > MAX_REGISTER_BYTES:
+ raise RegisterError(f"{source}: larger than {MAX_REGISTER_BYTES} bytes")
+ text = body.decode("utf-8", errors="replace")
+ try:
+ data = yaml.safe_load(text)
+ except yaml.YAMLError as exc:
+ raise RegisterError(f"{source}: not YAML: {exc}") from exc
+ if not isinstance(data, dict) or data.get("schema") != REGISTER_SCHEMA:
+ raise RegisterError(f"{source}: not a {REGISTER_SCHEMA} document")
+ return data
+
+
+def stat_values(register: dict) -> dict[str, int]:
+ """The expect value of every claim in STATS. Each must be a whole number."""
+ claims = {c.get("id"): c for c in register.get("claims") or [] if isinstance(c, dict)}
+ values = {}
+ for claim_id, _ in STATS:
+ expect = (claims.get(claim_id) or {}).get("expect")
+ if isinstance(expect, bool) or not isinstance(expect, int) or expect < 0:
+ raise RegisterError(f"{claim_id}: expect is not a whole number: {expect!r}")
+ values[claim_id] = expect
+ return values
+
+
+def render_block(values: dict[str, int], indent: str) -> str:
+ lines = [indent + START]
+ for claim_id, label in STATS:
+ lines.append(
+ f'{indent}{values[claim_id]}'
+ f'{label}'
+ )
+ lines.append(indent + END)
+ return "\n".join(lines)
+
+
+def typed_counts(path: Path) -> list[str]:
+ """Counts typed into a page outside the stat block and samples."""
+
+ def blank(match: re.Match) -> str:
+ return "\n" * match.group(0).count("\n") # keep line numbers
+
+ text = PRE.sub(blank, BLOCK.sub(blank, path.read_text(encoding="utf-8")))
+ return [
+ f"{path.relative_to(ROOT)}:{lineno}: {match.group(0)!r}"
+ for lineno, line in enumerate(text.splitlines(), 1)
+ for pattern in (TYPED_COUNT, BADGE_COUNT)
+ for match in pattern.finditer(line)
+ ]
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--check", action="store_true",
+ help="change nothing; exit 1 if the block is stale or a count is typed")
+ parser.add_argument("--register", default=os.environ.get(REGISTER_ENV) or REGISTER_URL_DEFAULT,
+ help="claims register: a file path or an https URL")
+ args = parser.parse_args(argv)
+
+ try:
+ values = stat_values(read_register(args.register))
+ except RegisterError as exc:
+ print(f"sync_site_numbers: {exc}", file=sys.stderr)
+ return 2
+
+ html = INDEX.read_text(encoding="utf-8")
+ match = BLOCK.search(html)
+ if match is None:
+ print("sync_site_numbers: index.html has no stat:start/stat:end block", file=sys.stderr)
+ return 2
+ updated = html[: match.start()] + render_block(values, match.group("indent")) + html[match.end():]
+ summary = ", ".join(f"{claim_id} {values[claim_id]}" for claim_id, _ in STATS)
+
+ if args.check:
+ problems = []
+ if updated != html:
+ problems.append("index.html: the stat block is stale; run scripts/sync_site_numbers.py")
+ for page in PAGES:
+ problems += [f"{hit}: a count typed outside the stat block" for hit in typed_counts(page)]
+ for problem in problems:
+ print(problem, file=sys.stderr)
+ if problems:
+ return 1
+ print(f"site numbers match the claims register: {summary}")
+ return 0
+
+ if updated == html:
+ print(f"index.html: stat block already current: {summary}")
+ else:
+ INDEX.write_text(updated, encoding="utf-8")
+ print(f"index.html: stat block updated: {summary}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/test_sync_site_numbers.py b/tests/test_sync_site_numbers.py
new file mode 100644
index 0000000..27f84b2
--- /dev/null
+++ b/tests/test_sync_site_numbers.py
@@ -0,0 +1,111 @@
+"""Tests for scripts/sync_site_numbers.py. Run: python -m pytest tests -q"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+import sync_site_numbers as sync # noqa: E402
+
+VALUES = {"C-01": 87, "C-03": 43, "C-06": 12, "C-07": 120, "C-08": 6}
+
+PAGE = """
+
+Built-in attack chains.
+Pattern 40 rules . always active
+Findings: 2 Chains: 1
+
+"""
+
+
+def write_register(path: Path, values: dict, schema: str = sync.REGISTER_SCHEMA) -> Path:
+ claims = [{"id": cid, "expect": value} for cid, value in values.items()]
+ path.write_text(yaml.safe_dump({"schema": schema, "claims": claims}), encoding="utf-8")
+ return path
+
+
+@pytest.fixture
+def site(tmp_path, monkeypatch):
+ (tmp_path / "docs").mkdir()
+ (tmp_path / "index.html").write_text(PAGE, encoding="utf-8")
+ (tmp_path / "docs" / "index.html").write_text("Detection rules
\n", encoding="utf-8")
+ (tmp_path / "README.md").write_text("Published records.\n", encoding="utf-8")
+ monkeypatch.setattr(sync, "ROOT", tmp_path)
+ monkeypatch.setattr(sync, "INDEX", tmp_path / "index.html")
+ monkeypatch.setattr(sync, "PAGES", tuple(tmp_path / p for p in ("index.html", "docs/index.html", "README.md")))
+ return tmp_path
+
+
+def test_writes_every_value_then_check_passes(site):
+ register = write_register(site / "register.yaml", VALUES)
+ assert sync.main(["--register", str(register)]) == 0
+ html = (site / "index.html").read_text(encoding="utf-8")
+ for claim_id, label in sync.STATS:
+ assert f'stat-num">{VALUES[claim_id]}