diff --git a/CHANGELOG.md b/CHANGELOG.md index 56416a00..48f09005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ ### 新增 Added + - **Scanner self-declared ranges: source + eval truth-layer exclusion + label pinning (issues #212 #213 #214, docs closeout #215; scanner_ranges batch; census docs/research/2026-10-26-scanner-ranges-census.md, ADR 0011)**: new `backend/ipdb/_sources/scanner_ranges.py` (cdn_edges-shaped _FEEDS multi-publisher table) ingests 66 MISP warninglists non-nt scanning lists (CC0, CIRCL mirror, ~29,070 CIDR v4+v6 mixed, date-versioned daily) + Palo Alto Xpanse reference-page direct read (9 CIDR — the only major absent from MISP); OpenFilters/internet-scanners (same upstream) deliberately NOT wired — one route per truth, echo discipline; GreyNoise API-only stays out (red line); evidence face is the user-ruled accusation form: Evidence(classification="scanner", verdict="malicious", native_categories=["研究扫描 · <厂商/机构>(自报段)"], r=0.90, stale_days=7, provenance_tier B) — malicious is the behavioral-type assertion (词表 "aggressive scanning"), intent lives in the native label, adjudication stays with the user; independent two-vote corroboration with behavior witnesses (sw_scanners) is legal (observation line independent, no lineage overlap). Eval truth layer (ADR 0011, deliberate product/eval asymmetry = feature): assembly-side exclusion — `layer_builders.scanner_ranges_prefixes` reads prefixes from the raw middleware CSV (backend/data/scanner_ranges.csv, first column, never reverse-parsed from LMDB; missing CSV raises FileNotFoundError — the exclusion face must not silently shrink; empty merged table parks DEGRADED via truth_layer scanner_ok) feeding the existing `build_truth_layer(scanner_prefixes=)` hook; `corpus.build_benchmark` skips the scanner_ranges source and strains every stratum through `apply_scanner_filter` post-cap (宁少算, no backfill; no-source / missing-CSV degenerates bit-identical to the status quo); S12 corpus versioning and the S17 truth-fingerprint-change trend-reset note are pinned by tests (40.124.175.29 two-arm: member diff exact, removed_by_scanner_filter 0→1, both scanlist fingerprints change); the truth-side ops runbook rewire (legacy censys+Xpanse+Shadowserver hand-merged asset → scanner_ranges loader) is measured advice left for the ops round. Product surface: zero new UI mechanism — the native label rides the existing native_categories chip chain (_merge.py:306-331 → _api_models details → SourceDetailRow chip), pinned by double-ended contract tests (real lookup() full chain on 40.124.175.29: malicious verdict stands, label in details, corroborated=False, verdict_conflict stays dark; LookupResultOut outbound shape re-pinned; frontend chip full-literal visible); tamper-red proven both ends. Gates: source tests 11/11 offline (66+1 name-set verbatim vs spec; negative assertions: no OpenFilters, no -nt-, no GreyNoise), backend eval 489/0, backend targeted 1791/0 (source+core) and 111/0 (label chain), frontend vitest 11/11, full backend 2251 passed / 9 skipped (4 dispositions: 2 legit metadata-snapshot updates landed with the ticket, 1 pre-existing baseline failure unchanged, 1 load-flake single-run green). + - **扫描器自报段三件套:源接入+eval 真值层排除+标签钉测(issues #212 #213 #214,文档收口 #215;scanner_ranges 批;普查 docs/research/2026-10-26-scanner-ranges-census.md,ADR 0011)**:新增 `backend/ipdb/_sources/scanner_ranges.py`(cdn_edges 同构 _FEEDS 多发布者表)接入 MISP warninglists 66 个非 nt 扫描列表(CC0,CIRCL 镜像,~29,070 条 CIDR v4/v6 混排,日期版本日更)+ Palo Alto Xpanse 参考页直读(9 CIDR——唯一不在 MISP 的大厂);OpenFilters/internet-scanners 同上游刻意不接——每真值一路,防回声;GreyNoise API-only 红线外;证据面=用户裁定的指控形态:Evidence(classification="scanner", verdict="malicious", native_categories=["研究扫描 · <厂商/机构>(自报段)"], r=0.90, stale_days=7, provenance_tier B)——malicious 是行为类型断言(词表 "aggressive scanning"),意图活在 native 标签里,改判权留用户;与行为证人(sw_scanners)独立两票 corroborated 合法(观测线独立,无谱系重叠)。eval 真值层(ADR 0011,产品/eval 刻意不对称=特性):装配侧排除——`layer_builders.scanner_ranges_prefixes` 从原始中间件 CSV(backend/data/scanner_ranges.csv,首列)取前缀表(勿从 LMDB 反解;缺 CSV raise FileNotFoundError——排除面不得无声缩水;合并表空由 truth_layer scanner_ok 落 DEGRADED),喂现成 `build_truth_layer(scanner_prefixes=)` 挂点;`corpus.build_benchmark` 跳过 scanner_ranges 源+封顶后逐分层过 `apply_scanner_filter`(宁少算不回补;无源/缺 CSV 与现状逐位一致退化);S12 语料版本与 S17 truth 指纹换版趋势重置注记以测试钉住(40.124.175.29 两臂:成员差精确、removed_by_scanner_filter 0→1、双 scanlist 指纹皆变);truth 侧 ops runbook 换源(legacy censys+Xpanse+Shadowserver 手合资产→scanner_ranges loader)为测量建言,留 ops 轮人裁。产品面:零新 UI 机制——native 标签走既有 native_categories chip 链(_merge.py:306-331→_api_models details→SourceDetailRow chip),双端契约测试钉死(40.124.175.29 真 lookup() 全链:malicious 判决照旧、标签在 details、corroborated=False、verdict_conflict 不点亮;LookupResultOut 出海形状复钉;前端 chip 全文字面量可见);双端 tamper-red 红证。门:源测试 11/11 offline(66+1 名单与 spec 逐字;负断言:无 OpenFilters/无 -nt-/无 GreyNoise)、后端 eval 489/0、后端定向 1791/0(源+core)与 111/0(标签链)、前端 vitest 11/11、后端全量 2251 过/9 跳(4 挂处置:2 快照合法更新随票落地、1 基线同挂、1 负载 flake 单跑绿)。 - **Artifact-name table single source (issue #202; C1-C3 refactor batch, umbrella #201; pure refactor, disk names byte-identical)**: artifact filenames were raw string literals in three consumption layers (gates.mjs ~45 sites, steps.mjs ~31, selftest-pipeline's 17-name hand-copied list) — adding 09-budget.json in #199 meant hand-syncing three layers (live example); new `ops/agent/artifacts.mjs` exports `ARTIFACTS` (18 fixed names, frozen) + `ARTIFACT_NAMES` + the DEEP parameterized families (`deepArtifactName`/`deepTranscriptName` factories + `DEEP_ARTIFACT_RE`/`DEEP_TRANSCRIPT_RE` matchers); all three layers now take names from the table; a two-way closure sentinel pins table<->disk-set coherence (table gains an entry without a writer = red; a step emits a new artifact not in the table = red); golden/cases.json and the canned-faux artifact-name set are bit-identical pre/post (md5-equal); umbrella closure folds in the fourth write surface — finding-exit.mjs's EXITS_ARTIFACT now takes `ARTIFACTS.ANCHOR_SWEEP_EXITS` (C1-merger find). Known non-swept literals by spec design: faux.mjs transcriptName x3 and pipeline.mjs HEALTH_PIPELINE out/SKIP_WHEN (outside the spec's three layers; the disk-level sentinel backstops). Gates: selftest-pipeline 78/0 (tamper proof: injected fake table entry -> closure sentinel FAIL "工件缺:10-c1-tamper.json" -> restored green), t4 63/0, contract 18/0, golden 9/0, standard-check 47 OK, backend ops subset 271 passed 1 skipped. - **工件名表单源(issue #202;C1-C3 重构批,伞 #201;纯重构,磁盘工件名字节不变)**:工件文件名原是三层裸字符串(gates.mjs ~45 处/steps.mjs ~31 处/selftest-pipeline 17 名手抄清单)——#199 加 09-budget.json 时手抄三层就是活例;新 `ops/agent/artifacts.mjs` 导出 `ARTIFACTS`(18 固定名 frozen)+`ARTIFACT_NAMES`+DEEP 参数族(deepArtifactName/deepTranscriptName 工厂+DEEP_ARTIFACT_RE/DEEP_TRANSCRIPT_RE 匹配面);三层全改从名表取值;双向闭包哨钉名表↔磁盘集合自洽(名表加项无写出面=红,步面新工件不入表=红);golden/cases.json 与罐装 faux 工件名集合重构前后位等价(md5 同);伞收口补入第四写面——finding-exit.mjs 的 EXITS_ARTIFACT 改取 `ARTIFACTS.ANCHOR_SWEEP_EXITS`(C1 merger 发现)。spec 设计内留白:faux.mjs transcriptName ×3+pipeline.mjs HEALTH_PIPELINE out/SKIP_WHEN 字面量(三层措辞之外;磁盘级哨兜底)。门:selftest-pipeline 78/0(红证:名表注入假名→闭包哨 FAIL "工件缺:10-c1-tamper.json"→还原回绿)、t4 63/0、contract 18/0、golden 9/0、standard-check 47 OK、backend ops 子集 271 passed 1 skipped。 - **Monthly-admission refusal block + ctx shape consolidated to single sources (issues #203 #204; C1-C3 refactor batch, umbrella #201; pure refactor, zero behavior change — C2+C3 scope; C1's artifacts-name table lands with its own worker)**: C2 — the monthly kill-switch refusal block was verbatim-duplicated across both faces (agent.mjs runRound legacy face ↔ pipeline.mjs runPipeline, the #200 spec-axis finding); `ops/agent/pipeline/budget.mjs` now exports the single source: `refusalOf` (pure text face) + `admitOrRefuse` (stateful shell: monthlyAdmission reads → on refusal the `[admission]` log line + the ledger trace row via appendRefusalTrace, write failure logged never flipping the verdict) ; both faces rewire to it and only map their exits (legacy throws {code} → failure envelope exit 1; pipeline emits emitFail envelope exit 1) — refusal message / [admission] log / ledger row / exit codes byte-identical, pinned by a new selftest-pipeline sentinel against a real ledger tree; the monthly-cap tolerance doctrine is centralized as the single annotation at admitMonthly (exact comparison, no ε — equality passes, spec-pinned "only > cap rejects"; the +1e-9 dust-tolerance precedent stays on the round-budget face and must never leak here; if a monthly ε is ever added, admitMonthly is the only legal site). C3 — runResume's hand-rebuilt ctx had drifted from mkCtx's shape with only human memory holding them together; pipeline.mjs now exports `ctxShape` — the ctx field-list single constructor; mkCtx passes validated fresh values, runResume passes state.json-recovered values with missing knobs re-read from env (budget_knobs/model_face fallbacks unchanged; budget_cny stays unvalidated and un-filled — an old round's undefined passes through verbatim); knob validation stays in mkCtx exactly as-is (not moved, not loosened — ctxShape does zero validation by design), state passes by reference, modelCalls/dupRegistry/usage are fresh per construction; two new sentinels: the ctxShape closed key-set (23 keys incl. the ctxPaths four) and the unique-construction-site check ("dupRegistry: new Map()" / "modelCalls: 0" / "usage: { input: 0" each appear exactly once in pipeline.mjs — a hand-rebuilt field list recurring is red). Gates: selftest-pipeline 81/0 (baseline 78; red proofs: unimplemented shell 78/1 "admitOrRefuse is not a function", tolerance flip <=→< reddened the equality-admits pin, removing ctxShape's breakerKnobs reddened the closed-set sentinel), t4 63/0, contract 18/0, golden 9/0 (canned face doubles as the bit-equivalence spot check), standard-check 47 OK (挂点31/建议10), backend ops subset 271 passed 1 skipped, legacy smoke exit-2 paths unchanged. diff --git a/GLOSSARY.md b/GLOSSARY.md index 75969ea7..8b2afb96 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -21,6 +21,7 @@ - **合法空**:bare-IP 多列表 feed 中 HTTP 200 + 空 body 的子列表——发布方「当前无此类条目」的声明:落 0 字节文件、下一 epoch 出清该列表条目、不计失败。仅限 200;404/超时/锚空仍是失败。firehol 等头部化格式不适用(空文件无头即坏)。锚:_sources/blocklist_de.py(download) - **活性锚**:同批下载中证明出版方存活的列表(blocklist_de = all.txt 聚合);锚空 = 本轮零替换 + 记真失败(fail-closed),防灾难性全空清库。判例:blocklist_de ircbot 2026-10-15(grill Q1-Q3)。锚:_sources/blocklist_de.py(download) - **载入期基础设施过滤**:聚合类源 rebuild 载入面排除 curated 公共基础设施单址集——聚合器转载的指控对公认解析器不构成行为证据,排除在投影层(数据文件零改写,无损管线不破)。真源=infra_services curated 集(单真相,勿查可能纪元落后的 LMDB);基类默认关,事故源显式 opt-in(infra_noise_filter=True);宁少算=仅精确单址命中,含 infra 的更大 CIDR 不排;IP_RADAR_INFRA_NOISE_FILTER=0 + 重建即复原,排除计数+条目日志化。判例:8.8.8.8(myip_ms 单源)/114.114.114.114(cps+ziyadnz 假印证)/1.1.1.1(cps)三公共解析器误报(2026-10-24 事故,ADR 0008 发现固化;哲学同族=资产弃权)。锚:_sources/_infra_filter.py +- **自报段**:IP 段由扫描主体自己公布、经机构镜像(MISP warninglists,CC0;Xpanse 直读)到达的厂商声明——身份证据非行为观测;镜像同源(OpenFilters/internet-scanners)只接一路防回声,与三方自采观测面(sw_scanners)无谱系重叠;兼作 eval 真值层排除清单(ADR 0011)。判例:scanner_ranges 接入 grill 2026-10-26。锚:backend/ipdb/_sources/scanner_ranges.py:129(下载面 _FEEDS 66+1 表在 :119,harvest 证人 :150) ## 证据 @@ -31,6 +32,7 @@ - **benign(两层,勿混)**:①指控缺席 = 弃权——观测级弃权拼写 verdict=""(R17A-1,序列化特判保留,勿兑底 malicious),顶层无指控观测时缺省序列化 verdict="benign"、confidence=0(_types.threat_summary);不投票、不亮黄灯,是「无证据」不是「判良性」——弃权绝不作良性背书(见「无证据≠清白」)。②良性断言 = 非指控信号源明确断言 benign(infra_services 升格为首个此类源,#188:curated 公共 DNS/NTP 单址双面 harvest——service 资产面照旧+classification_type=服务角色(dns/ntp)+verdict="benign";弃权拼写 "" 期作废);与指控同场才算真对立(verdict_conflict=True——良性断言 type-agnostic 注入每个指控组同场,注入面在 _registry.lookup);宁少算:良性证人不投票(组 conf/顶层重融合不变)、不压制指控 verdict(precedence 恒指控胜)、不产 STIX Indicator;conflict 留痕人裁不自动压制(锚巡检 conflict 投影/finding 出口 issue 标记);无指控组时以自身 ctype 成纯良性面(verdict=benign conf≈r,区别于①弃权 benign/0)。锚:_evidence.py(to_dict 弃权特判)、_types.py(threat_summary)、_merge.py(_assess_classification)、_registry.py(lookup 分组注入)、_sources/infra_services.py(harvest 双面) - **资产弃权**:源只陈述基础设施属性(proxy/hosting/tor),不构成滥用指控——verdict="" 不计票、不产 STIX Indicator,仅走 attributes 通道。判例:cloud_ranges(首创)→ip2proxy DCH(SM-F2)→ip2proxy VPN/PUB/TOR(ADR 0001,2026-10-12)。锚:_sources/ip2proxy.py(_proxy_evidence) - **行为源**:以观测到的滥用行为(扫描/爆破/C2 回连/举报)立证的源,如 bruteforce/ciarm/abuseipdb;与资产属性源相对——「挂着代理」由资产弃权陈述,「在作恶」由行为源指控。锚:_sources/bruteforce.py +- **公开研究扫描**:scanner 分类的自报段证人形态——verdict=malicious 是行为类型断言(词表 "aggressive scanning"),非独立过错指控,意图活在 native 标签「研究扫描 · <厂商/机构>(自报段)」里;与行为源指控构成独立两票(观测线独立,corroborated 合法);改判权留用户,非良性断言(自报不构成良性背书,无证据≠清白)。判例:grill Q3 2026-10-26(用户裁「还是恶意但打标签,用户自己评估」)。锚:backend/ipdb/_sources/scanner_ranges.py:150-161(harvest 构造 native_categories=["研究扫描 · <厂商/机构>(自报段)"] :161) - **无证据≠清白**:conf 0 = 无源命中,不是"干净";缺数据直说缺。锚:_api_models.py(FieldOut) ## 融合 diff --git a/NEEDS.md b/NEEDS.md index 454bf55b..3df96fe4 100644 --- a/NEEDS.md +++ b/NEEDS.md @@ -24,7 +24,7 @@ | ASN 业务类型(ISP/content/edu) | 2026-10-10 | 候选 | CAIDA as-classification 已死(官方 legacy、下载关闭,2026-10-10 查证);其真值本就是 PeeringDB 自报 info_type → 正路 = peeringdb 既有源扩展(org/net 端点 join,闸门判例),立项另裁 | | 攻击者 IP 第一方测量(蜜罐/传感网) | 2026-10-10 | 已闭 | knock_knock 已交付(PR #104,blacklist=binarydefense 双生判例) | | DDoS 攻击源 | 2026-10-10 | 已闭 | rtbh 已交付(PR #104,ddos 槽首个专职证人;许可用户签核 2026-10-10) | -| IP 是否研究扫描器(Censys/Shodan 等已知扫描段) | 2026-10-12 | 已立项（2026-10-12 eval 闭环批裁：真值层剔除同源语料污染，产品面同型呈现） | 官方公布段静态文件→attributes 通道，cdn.notice 同构判例；兼作威胁真值层排除清单（语料版本随附） | +| IP 是否研究扫描器(Censys/Shodan 等已知扫描段) | 2026-10-12 | 已交付待合(SDD 批 PR #216:#212 源接入+#213 eval 排除+#214 标签钉测+#215 文档收口;ADR 0011;报告 docs/research/2026-10-26-scanner-ranges-census.md) | 官方公布段静态文件→attributes 通道,cdn.notice 同构判例;兼作威胁真值层排除清单(语料版本随附);通道裁定=MISP warninglists -scanning 66 列表(CC0)+Xpanse 直读,80.2% 池内污染实测,grill 全裁 2026-10-26(恶意保持+「公开研究扫描/自报段」标签人裁;eval 语料装配侧排除,ADR 0011;单批三件套);truth ops runbook 换源接线(legacy censys+Xpanse+Shadowserver 手合资产→scanner_ranges loader)留 ops 轮人裁 | | 我们输出的 verdict 在真实执法点(阻断/检测消费端)的 TP/FP 表现 | 2026-10-12 | open | 业界金标准 = 消费端反馈回流(ThreatConnect CAL 社区众包/MISP sightings/GTI 分析师纠正,docs/research/2026-10-12-industry-loop-practices.md);前置 = 产品需有消费 verdict 并能回报处置结论的执法点 + 查询隐私裁决,2026-10-12 用户记「后续想做」;本批以蜜罐锚+纠错检验作无遥测近似 | | 土耳其政府 SGB 情报(siberguvenlik.gov.tr/api/address/index) | 2026-10-10 | open | JSON API,本轮未验活未取样;与 RTBH 同区域第二票,区域扩展时再验 | | geo_asn/attributes 通道源的接入关读数口径(现走硬门得零信息报告) | 2026-10-13 | open | 对抗审查第1轮:读数覆盖域=threat/asset 两轴(verdict N/A-ASSET 早退+metrics 仅遍历 classifications),geo_asn 现役 7 源即此态;标量轴/排除清单覆盖率口径未裁,待用户(扩张 Q4/Q18 gate 覆盖域须裁) | diff --git a/backend/ipdb/_eval/corpus.py b/backend/ipdb/_eval/corpus.py index cc5792fd..09f98397 100644 --- a/backend/ipdb/_eval/corpus.py +++ b/backend/ipdb/_eval/corpus.py @@ -11,6 +11,13 @@ from dataclasses import dataclass, field, asdict from ipaddress import IPv4Address +# SR-T2 D2(#213 / ADR 0011): benchmark 装配侧剔除自报段 —— 取表走 +# layer_builders.scanner_ranges_prefixes(原始 CSV 直读), 段匹配复用 +# truth_layer.apply_scanner_filter; import 方向 corpus → truth_layer/ +# layer_builders 单向无环(两模块均只 import .assets)。 +from .layer_builders import scanner_ranges_prefixes +from .truth_layer import apply_scanner_filter + _IP_RE = re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}(?:/\d+)?\b") @@ -93,10 +100,27 @@ def build_benchmark(sources, per_type_n: int, `sources` are baseline sources (candidate excluded). Each threat source contributes IPs bucketed by its classification_type. benign/reserved strata are the small known lists below (fixed; grow the lists, not knobs). + + ADR 0011 装配侧剔除(#213): scanner_ranges 源不入采样宇宙(成员= + 厂商自报的段本身, 非指控观测, 入语料=自证污染); 其 CSV 在场时, + 采样/封顶后对 benchmark 每个分层过一遍自报段前缀 —— 其他 scanner + 源采出的段内 IP 同剔(census 2026-10-26 实测 80.2% 扫描器 IP 已在 + 自报段)。宁少算不回补; v6 前缀值域与 v4 永不重叠, 混排安全 + (truth_layer.apply_scanner_filter)。无 scanner_ranges 源或其 CSV + 缺失 → 行为与现状逐位一致(排除器不启动)。benign/reserved 是固定 + 小名单非采样面, 不动。 """ rng = rng or random.Random() bench: dict[str, list[str]] = {} + selfdecl: list[str] = [] for s in sources: + if getattr(s, "name", None) == "scanner_ranges": + # 排除前缀只从在场 CSV 载入(缺文件=无排除=现状; D1 loader 的 + # raise 语义属 truth 装配侧, 这里 is_file 前置走 noop 分支) + path = getattr(s, "_path", None) + if path is not None and path.is_file(): + selfdecl = scanner_ranges_prefixes(path) + continue ctype = getattr(s, "classification_type", None) if not ctype: continue @@ -106,6 +130,9 @@ def build_benchmark(sources, per_type_n: int, for k in list(bench.keys()): rng.shuffle(bench[k]) bench[k] = bench[k][:per_type_n] + if selfdecl: + for k in list(bench.keys()): + bench[k], _dropped = apply_scanner_filter(bench[k], selfdecl) return Corpus( benchmark=bench, benign=["8.8.8.8", "1.1.1.1", "9.9.9.9", "208.67.222.222"], diff --git a/backend/ipdb/_eval/layer_builders.py b/backend/ipdb/_eval/layer_builders.py index e38ce5f2..457115b6 100644 --- a/backend/ipdb/_eval/layer_builders.py +++ b/backend/ipdb/_eval/layer_builders.py @@ -4,6 +4,9 @@ 威胁真值层归 truth_layer.py(T4, 自带 rir/扫描/健康度三闸); 本模块产 其余三层资产, 全部经 assets.save_asset 落 /ips.txt+meta.json (版本化+指纹, 装配走 layered.assemble_four_layers 的 // 布局)。 +truth 侧唯一例外: scanner_ranges_prefixes(自报段排除清单供给, SR-T2 +#213 / ADR 0011)寄住在此 —— 供 ops 离线组装 scanlist 时取表, truth ETL +本体仍在 truth_layer.py。 - 中性层(spec §2.1 600 静态): neutral_ips.json 原样包装, 无采样 → 无 run_seed(W0 静态资产逐轮恒定, S36 只辖采样路径)。 @@ -188,6 +191,32 @@ def build_consensus_asset(out_dir: Path, *, version: str, run_seed: int, return meta +def scanner_ranges_prefixes(csv_path: Path) -> list[str]: + """自报段前缀表供给: 逐行读 scanner_ranges 中间件 CSV 首列(SR-T2 #213)。 + + ADR 0011: eval 真值语料装配侧剔除自报段成员 —— ops 离线组装 scanlist + 时以 legacy 清单 + 本表喂 build_truth_layer(scanner_prefixes=...), + 换版经 scanlist_fingerprint(S12)/趋势重置(S17)自动留痕。原始 CSV + 直读(无损管线: 原始证据可回算), 勿从 LMDB 反解 CIDR(EXPLORE §4)。 + + 行格式 cidr,vendor: 取首列(逗号前)strip 后 token; 空/纯空白行跳过; + token 原样透传(含裸 IP 与坏 token) —— 归一与坏值拦截归 + apply_scanner_filter 侧 ip_network(「坏快照不许静默缩水」, + parse_rir_delegated 同款哲学)。CSV 缺失 → FileNotFoundError: 显式 + 索取的排除清单缺席是装配错误, 静默回空表会与 legacy 合表后无声 + 缩小排除面(语料悄然重染); 纯空表组合另有 gates_health + scanner_ok 的 DEGRADED 拒出版兜底(truth_layer S9)。benchmark 装配 + 侧(corpus.build_benchmark)要「缺文件即无排除」语义, 自行 is_file + 前置判断。 + """ + out: list[str] = [] + for line in Path(csv_path).read_text(encoding="utf-8").splitlines(): + token = line.split(",", 1)[0].strip() + if token: + out.append(token) + return out + + def build_all_monthly(assets_root: Path, *, version: str, run_seed: int, cloud_ranges: list[str], resolver_ips: list[str], source_keysets: dict[str, list[str]]) -> dict[str, AssetMeta]: diff --git a/backend/ipdb/_sources/scanner_ranges.py b/backend/ipdb/_sources/scanner_ranges.py new file mode 100644 index 00000000..b79898e5 --- /dev/null +++ b/backend/ipdb/_sources/scanner_ranges.py @@ -0,0 +1,198 @@ +"""scanner_ranges (扫描器厂商自报段) — Source subclass, cdn_edges 同构. + +67 路单源:66 个 MISP warninglists `*-scanning` 家族(CC0, CIRCL;URL 走 +raw.githubusercontent 一跳镜像)+ Xpanse 官方页直读(cortex-docs +scanning-activity)。实测 2026-10-26 普查(docs/research/2026-10-26-scanner-ranges-census.md): +66 列表 29,070 CIDR(5,919 v6,22 裸 IP);污染探针 80.2% 已被本地池标恶意 +(N=400, seed 42)→ 立项裁定 = eval 真值层剔除成员(SR-T2),本源只作指控票。 + +证据形态(2026-10-26 用户裁定, ADR 0011 / GLOSSARY「自报段」「公开研究扫描」; +同日 spec 补定 native_categories 路线, #212 评论): +Evidence(classification="scanner", verdict="malicious", +native_categories=["研究扫描 · <厂商/机构>(自报段)"])——标签载体用 native_categories +(f3csystems.py:53 先例 → _merge.py:306-331 detail dict → _api_models.py:56 +透传 → 前端 SourceDetailRow.tsx 内联 chip),不用 native_types(它只喂 +ASSET_SLOTS AssetStatement, scanner 无槽,证人行渲染不出来)。非 service 资产槽 +(cloudzy 判例),非良性断言(拒 benign②);verdict=malicious 是行为类型断言 +("aggressive scanning" 词表),与 sw_scanners 等行为源构成独立两票, +corroborated +1 合法。 + +r=0.90 统一:自报段=厂商官方声明(高可信),但 MISP 是一跳镜像(对比 +cdn_edges 出版商直连 0.95 折档);Xpanse 虽官方直读,同源单值统一 0.90, +防同源双权谱系噪声。stale_days=7(慢变段表,cf. cdn_edges)。 +provenance_tier="B"(MISP 一跳镜像主导;Xpanse 官方页同源单值不拆档—— +threat 类必声明, test_eval_source_meta 硬规);OpenFilters 镜像不接(56 列表 +上游同源,只接一路防回声)、-nt- 22 列表不接(CIRCL 观测提取非自报)、GreyNoise +API-only 红线拒(spec 定案)。 + +download() 拉全部 67 路并写合并中间件 `scanner_ranges.csv` (cidr,vendor) +(与 cdn_edges (cidr,provider) 同构);harvest() 映射为 Evidence。厂商名从 +列表自身 MISP 元数据(name 字段)折短;双族由 rebuild_dual_family 写入侧 +自动分派(list 数组 v4/v6 混排,裸 IP 收作 /32)。 +""" +import ipaddress +import json +import re + +from ._download import atomic_write_bytes +from .._source_base import Source +from .._evidence import Evidence + +_MISP_URL_TMPL = ("https://raw.githubusercontent.com/MISP/misp-warninglists/" + "main/lists/{name}/list.json") +_XPANSE_URL = ("https://cortex-docs.paloaltonetworks.com/cortex-xpanse/" + "reference/scanning-activity") + +# (list name, 厂商/机构) — 厂商名折短自各列表 list.json 自身的 name 元数据 +# (/tmp/misl_*.json 缓存, 2026-10-26 实测; 全 66 条, 勿遗漏)。 +_MISP_LISTS = ( + ("a10networks-scanning", "A10 Networks"), + ("academyforinternetresearch-scanning", "Academy for Internet Research"), + ("adscore-scanning", "Adscore"), + ("alphastrike-scanning", "Alpha Strike Labs"), + ("binaryedge-scanning", "BinaryEdge"), + ("bitsight-scanning", "BitSight"), + ("bufferoverrun-scanning", "BufferOver.run"), + ("censys-scanning", "Censys"), + ("circl-security-scanning", "CIRCL"), + ("circl-ssh-scanning", "CIRCL"), + ("criminalip-scanning", "Criminal IP"), + ("crowdstrikefalconsurface-scanning", "CrowdStrike Falcon Surface"), + ("cybcube-scanning", "CyberCube"), + ("cybergreen-scanning", "CyberGreen Institute"), + ("cyberok-scanning", "CyberOK"), + ("cyberresilience-scanning", "CyberResilience"), + ("cypex-scanning", "Cypex"), + ("dataplane-scanning", "Dataplane"), + ("deepfield-scanning", "Nokia Deepfield"), + ("detectify-scanning", "Detectify"), + ("edgewatch-scanning", "Edgewatch"), + ("eset-scanning", "ESET"), + ("fofa-scanning", "FOFA"), + ("groupib-scanning", "Group-IB"), + ("hackertarget-scanning", "Hacker Target"), + ("hadrian-scanning", "Hadrian"), + ("hunter-scanning", "Hunter"), + ("infrawatch-scanning", "Infrawatch"), + ("internetcleanup-scanner", "Internet Cleanup Foundation"), + ("internetmeasurementresearch-scanning", "Internet Measurement Research"), + ("internetmetrics-scanning", "Internet Metrics"), + ("internetresearchproject-scanning", "Internet Research Project"), + ("internettl-scanning", "InterneTTL"), + ("intrinsec-scanning", "Intrinsec"), + ("intruder-scanning", "Intruder"), + ("ipinfo-scanning", "IPinfo"), + ("ipip-scanning", "IPIP"), + ("ki3-scanning", "KI3"), + ("knoq-scanning", "Knoq"), + ("leakix-scanning", "LeakIX"), + ("mandiantasm-scanning", "Mandiant ASM"), + ("modat-scanner", "Modat"), + ("netlas-scanning", "Netlas"), + ("netscout-scanning", "NETSCOUT"), + ("netsecscan-scanning", "NetSecScan"), + ("odin-scanning", "Cyble ODIN"), + ("onlyscans-scanning", "ONLYSCANS"), + ("onyphe-scanner", "Onyphe"), + ("pocnroll-scanning", "pocnroll"), + ("probethenet-scanning", "ProbeTheNet"), + ("quake-scanning", "360 Quake"), + ("rapid7-scanning", "Rapid7"), + ("recordedfuture-scanning", "Recorded Future"), + ("recyber-scanning", "Recyber"), + ("researchscanner-scanning", "Research Scanner"), + ("rootevidence-scanning", "Root Evidence"), + ("securityresearch-scanning", "securityresearch.net"), + ("shadowforce-scanning", "Shadowforce"), + ("shodan-scanning", "Shodan"), + ("stretchoid-scanning", "Stretchoid"), + ("tenableasm-scanning", "Tenable ASM"), + ("university-scanning", "University scanners"), + ("visionheight-scanning", "VisionHeight"), + ("xmco-scanning", "XMCO"), + ("zern-scanning", "ZERN"), + ("zoomeye-scanning", "ZoomEye"), +) + +# (name, url, vendor, fmt) — cdn_edges (provider,url,fmt) 同构 +fmt: +# MISP 66 路共享一个 URL 模板(list.json),Xpanse 走 HTML 直读。 +_FEEDS = tuple( + (name, _MISP_URL_TMPL.format(name=name), vendor, "misp") + for name, vendor in _MISP_LISTS +) + (("xpanse", _XPANSE_URL, "Palo Alto Xpanse", "xpanse"),) + +# HTML 直读用:最宽形态护栏(IPv4/IPv6 字符集 + 前缀长),候选再经 +# ip_network 语义闸过滤(HTML 噪声进不来;坏前缀长 ValueError 弃)。 +_CIDR_TOKEN_RE = re.compile(r"[0-9A-Fa-f:.]+/\d{1,3}") + + +class ScannerRangesSource(Source): + name = "scanner_ranges" + category = "threat" + filename = "scanner_ranges.csv" # combined intermediate written by download() + fields = ("is_malicious",) + classification_type = "scanner" + verdict = "malicious" + stale_days = 7 # 慢变段表(cf. cdn_edges) + reliability = 0.90 # 自报段官方声明 × 一跳镜像折档(spec 定案) + provenance_tier = "B" # MISP 一跳镜像主导(threat 类必声明,test_eval_source_meta) + authoritative_for = () # corroboration only (SM-F1, sw_scanners 同款) + + def download(self, token=None) -> None: + self._data_dir.mkdir(parents=True, exist_ok=True) + rows = [] + for _name, url, vendor, fmt in _FEEDS: + data = self._http_get(url) + rows.extend((cidr, vendor) for cidr in _parse(data, fmt)) + atomic_write_bytes(self._path, + "".join(f"{c},{v}\n" for c, v in rows).encode("utf-8")) + + def harvest(self): + for line in self._path.read_text().splitlines(): + if not line.strip(): + continue + cidr, vendor = line.split(",", 1) + yield cidr, Evidence( + classification_type=self.classification_type, + verdict=self.verdict, + reliability=self.reliability, + # 标签走 native_categories(f3csystems 先例,spec 补定): + # native_types 只喂 ASSET_SLOTS,scanner 无槽渲染不出。 + native_categories=[f"研究扫描 · {vendor}(自报段)"], + ) + + +def _norm_cidr(entry: str) -> str | None: + """归一为规范 CIDR 字符串;裸 IP 收作 /32(或 v6 /128),坏条目 → None。""" + try: + return str(ipaddress.ip_network(entry.strip(), strict=False)) + except ValueError: + return None + + +def _parse(data: bytes, fmt: str): + """Yield CIDR strings from one feed's raw bytes (both families). + + misp: list.json {"list": [...]} — v4/v6 混排(实测 5,919 v6),非字符串 + 与坏条目弃;裸 IP 经 _norm_cidr 收作 /32。 + xpanse: 官方页 HTML — _CIDR_TOKEN_RE 提候选(文档表格内的 9 CIDR), + ip_network 语义闸过滤页面噪声。 + """ + if fmt == "misp": + d = json.loads(data) + for entry in d.get("list") or []: + if not isinstance(entry, str): + continue + cidr = _norm_cidr(entry) + if cidr: + yield cidr + elif fmt == "xpanse": + text = data.decode("utf-8", errors="ignore") + found = [c for c in (_norm_cidr(t) for t in _CIDR_TOKEN_RE.findall(text)) if c] + # 零命中即页面改版嫌疑——炸响拒绝静默缩水(排除面同表,Spec 终审 P2; + # 宁多勿无:页面新增 CIDR 属合法更新,不设上限)。 + if not found: + raise ValueError( + "Xpanse page yielded 0 CIDR — layout change suspected, " + "refusing silent exclusion-face shrink") + yield from found diff --git a/backend/tests/core/test_metadata_derivation.py b/backend/tests/core/test_metadata_derivation.py index 32585a51..8f9fe45a 100644 --- a/backend/tests/core/test_metadata_derivation.py +++ b/backend/tests/core/test_metadata_derivation.py @@ -27,6 +27,7 @@ "cps_collected_iocs": "threat", # 波3 T3 CPS 自有聚合(2026-10-19,derived 聚合源) "ustc_blackip": "threat", # 波4 T1 USTC 校园自保黑名单(2026-10-19,非 derived) "myip_ms": "threat", # 波4 T2 myip.ms htaccess 黑名单(2026-10-19,非 derived) + "scanner_ranges": "threat", # SR-T1 扫描器厂商自报段(2026-10-26,MISP 66+Xpanse,非 derived) "cloudzy": "asset", # 波3 T2 Cloudzy VPS 段标签(2026-10-19,asset 轴非 derived) "ip2proxy": "asset", "tor_exits": "asset", "x4bnet_vpn": "asset", "proxyscrape": "asset", "infra_services": "asset", "cdn_edges": "asset", @@ -63,6 +64,7 @@ "cps_collected_iocs": 0.50, # 波3 T3 CPS 自有聚合(2026-10-19) "ustc_blackip": 0.55, # 波4 T1 USTC 校园自保黑名单(2026-10-19) "myip_ms": 0.55, # 波4 T2 myip.ms htaccess 黑名单(2026-10-19) + "scanner_ranges": 0.90, # SR-T1 扫描器厂商自报段(2026-10-26,MISP 镜像一跳折档) "cloudzy": 0.90, # 波3 T2 Cloudzy VPS 段标签(2026-10-19) "drb_ra": 0.50, "hookzof": 0.50, "thespeedx": 0.50, "protonvpn": 0.75, "nordvpn": 0.75, @@ -90,12 +92,12 @@ _PHANTOM_AXES = ("is_malicious", "is_hosting", "is_mobile") def test_categories_match_snapshot(): - # 50 = rir_abuse 入册后实测 len(SOURCE_CATEGORIES) + # 51 = scanner_ranges 入册后实测 len(SOURCE_CATEGORIES)(50 = rir_abuse 入册后口径) live = dict(r.SOURCE_CATEGORIES) assert live == OLD_CATEGORIES def test_reliability_match_snapshot(): - # 公开源快照全等锁死(43 源口径) + # 公开源快照全等锁死(44 源口径;43 = myip_ms 入册后口径) live = dict(m.SOURCE_RELIABILITY) assert live == OLD_RELIABILITY diff --git a/backend/tests/eval/test_eval_corpus.py b/backend/tests/eval/test_eval_corpus.py index ce3dd993..e6c56a45 100644 --- a/backend/tests/eval/test_eval_corpus.py +++ b/backend/tests/eval/test_eval_corpus.py @@ -1,5 +1,5 @@ # backend/test_eval_corpus.py -import json, tempfile, os, random +import ipaddress, json, tempfile, os, random from pathlib import Path from ipdb._eval.corpus import Corpus, sample_source_ips, build_benchmark, stable_seed @@ -80,6 +80,56 @@ def test_sample_respects_run_seed_over_rng(tmp_path): assert sample_source_ips(src, 100, random.Random(0), run_seed=43) != a +# ── SR-T2 D2: benchmark 装配侧剔除自报段(ADR 0011) ──────────── + +def test_build_benchmark_excludes_selfdeclared_members_all_strata(tmp_path): + # ①scanner_ranges 源不入采样宇宙(成员=厂商声明的段本身,非指控观测, + # 入语料=自证污染);②采样/封顶后逐分层剔自报段成员 —— 其他源采出的 + # 段内 IP 同剔(census 实测 80.2% 扫描器 IP 已在自报段)。逐 IP 判定用 + # ipaddress 独立算(非 apply_scanner_filter 回声,防 tautology)。 + sr_csv = tmp_path / "scanner_ranges.csv" + sr_csv.write_text("40.124.175.0/24,Stretchoid\n203.0.113.0/24,Xpanse\n") + sr = _FakeSource("scanner_ranges", sr_csv, classification_type="scanner") + sw_raw = tmp_path / "sw.txt" + sw_raw.write_text("40.124.175.29\n203.0.113.5\n198.51.100.7\n192.0.2.9\n") + sw = _FakeSource("sw_scanners", sw_raw, classification_type="scanner") + et_raw = tmp_path / "et.txt" + et_raw.write_text("40.124.175.30\n198.51.100.8\n") + et = _FakeSource("et", et_raw, classification_type="exploit-traffic") + bench = build_benchmark([sr, sw, et], per_type_n=10, rng=random.Random(0)) + nets = [ipaddress.ip_network(p) for p in + ("40.124.175.0/24", "203.0.113.0/24")] + for ips in bench.benchmark.values(): # 任一分层零自报段成员 + for ip in ips: + assert not any(ipaddress.ip_address(ip) in n for n in nets), ip + # 段外成员存活;scanner 分层只余 sw 的段外两个(sr 零贡献) + assert set(bench.benchmark["scanner"]) == {"198.51.100.7", "192.0.2.9"} + assert set(bench.benchmark["exploit-traffic"]) == {"198.51.100.8"} + + +def test_build_benchmark_without_scanner_ranges_source_unchanged(tmp_path): + # 回归臂:sources 无 scanner_ranges 源 → 行为与现状一致:段内 IP 照旧 + # 入语料(排除器只在自报段源在场时启动,ADR 0011 只接管在场源) + raw = tmp_path / "sw.txt" + raw.write_text("40.124.175.29\n198.51.100.7\n") + src = _FakeSource("sw", raw, classification_type="scanner") + bench = build_benchmark([src], per_type_n=10, rng=random.Random(0)) + assert set(bench.benchmark["scanner"]) == {"40.124.175.29", "198.51.100.7"} + + +def test_build_benchmark_scanner_ranges_missing_csv_noop(tmp_path): + # 回归臂:CSV 缺失(源未下载)→ 源跳过 + 排除器不启动 = 现状 + # (sample_source_ips 对缺文件本就返 [];前缀只从在场 CSV 载入, + # 不走 D1 的 raise 语义 —— loader 的 raise 是 truth 装配侧契约) + sr = _FakeSource("scanner_ranges", tmp_path / "nope.csv", + classification_type="scanner") + raw = tmp_path / "sw.txt" + raw.write_text("40.124.175.29\n198.51.100.7\n") + sw = _FakeSource("sw", raw, classification_type="scanner") + bench = build_benchmark([sr, sw], per_type_n=10, rng=random.Random(0)) + assert set(bench.benchmark["scanner"]) == {"40.124.175.29", "198.51.100.7"} + + def test_run_for_source_records_run_seed(tmp_path): # S36:报告 JSON 顶层 run_seed(缺省=secrets.randbits(64) 显式随机, # 不得回退源名派生)+ 采样指纹(被采键集 fingerprint,存 metrics 旁) diff --git a/backend/tests/eval/test_eval_layer_builders.py b/backend/tests/eval/test_eval_layer_builders.py index a8a70e0d..70544497 100644 --- a/backend/tests/eval/test_eval_layer_builders.py +++ b/backend/tests/eval/test_eval_layer_builders.py @@ -8,6 +8,8 @@ import json from pathlib import Path +import pytest + from ipdb._eval.assets import fingerprint, load_asset from ipdb._eval.layer_builders import ( CLOUD_SAMPLE_N, @@ -16,6 +18,7 @@ build_infra_asset, build_neutral_asset, non_vpn_resolvers, + scanner_ranges_prefixes, ) from ipdb._eval.layered import assemble_four_layers @@ -151,6 +154,47 @@ def test_infra_skips_v6_blocks_honest_fewer_and_counts(tmp_path): assert loaded.params["skipped_v6_blocks"] == 2 +# ── SR-T2 D1: scanner_ranges 前缀供给(truth 侧排除器) ────────── + + +def test_scanner_ranges_prefixes_reads_first_column_tokens(tmp_path): + # 逐行取首列(逗号前)strip:CIDR/裸 IP/v6 原样透传;空行与纯空白行跳过; + # 坏 token 不在 loader 拦(归 apply_scanner_filter 侧 ip_network 炸 —— + # 「坏快照不许静默缩水」:坏行混进真表时炸出来,不许静默缩成半张表) + csv = tmp_path / "scanner_ranges.csv" + csv.write_text( + "40.124.175.0/24,Stretchoid\n" + "2600:1f00::/24,Amazon\n" + "198.51.100.7,BareVendor\n" + "\n" + " \n" + "not-a-token,BadVendor\n" + "144.86.173.0/24,Palo Alto Xpanse\n", + encoding="utf-8", + ) + assert scanner_ranges_prefixes(csv) == [ + "40.124.175.0/24", "2600:1f00::/24", "198.51.100.7", + "not-a-token", "144.86.173.0/24", + ] + + +def test_scanner_ranges_prefixes_missing_csv_raises(tmp_path): + # 缺失裁定(implementer 定):显式索取的排除清单缺席 = 装配错误 → + # FileNotFoundError,不静默回空表(否则与 legacy scanlist 合表后排除面 + # 无声缩水,语料悄然重染);纯空表组合另有 gates_health scanner_ok + # 的 DEGRADED 拒出版兜底(truth_layer S9)。benchmark 侧要「缺文件即 + # 无排除」语义,由调用方(corpus.build_benchmark)自行 is_file 前置。 + with pytest.raises(FileNotFoundError): + scanner_ranges_prefixes(tmp_path / "nope.csv") + + +def test_scanner_ranges_prefixes_blank_csv_returns_empty(tmp_path): + # 空表(文件在、零 token)如实回空:诚实输入,下游闸兜底 + csv = tmp_path / "scanner_ranges.csv" + csv.write_text("\n \n", encoding="utf-8") + assert scanner_ranges_prefixes(csv) == [] + + def test_build_all_monthly_layout_matches_four_layer_assembler(tmp_path): root = tmp_path / "assets" metas = build_all_monthly(root, version="v1", run_seed=99, diff --git a/backend/tests/eval/test_eval_monthly.py b/backend/tests/eval/test_eval_monthly.py index 0a78dd20..9d1446dd 100644 --- a/backend/tests/eval/test_eval_monthly.py +++ b/backend/tests/eval/test_eval_monthly.py @@ -8,6 +8,7 @@ """ import gzip import json +import tempfile from datetime import date, timedelta from pathlib import Path @@ -393,6 +394,37 @@ def test_md_reset_note_excludes_rolling_consensus(): assert "滚动语料" in md2 +def test_md_reset_note_truth_two_arm_scanner_ranges_join(): + """SR-T2 D3(#213 / ADR 0011): S17 趋势重置注记 —— ops 两臂真指纹换版。""" + with tempfile.TemporaryDirectory() as td: + from ipdb._eval.layer_builders import scanner_ranges_prefixes + from ipdb._eval.truth_layer import build_truth_layer + tmp = Path(td) + csv = tmp / "scanner_ranges.csv" + csv.write_text("40.124.175.0/24,Stretchoid\n2600:1f00::/24,Amazon\n", + encoding="utf-8") + # 与 test_eval_truth_layer._BIG_RANGES 同构: 10.0.0.0 起步 10k 段 + base = 167_772_160 # int("10.0.0.0") + big = [(base + i * 256, 256, "20000101") for i in range(10_000)] + raw = ["40.124.175.29", "5.1.0.5", "9.9.9.9"] + legacy = ["10.0.0.0/8"] + a_dir, b_dir = tmp / "a", tmp / "b" + meta_a = build_truth_layer(raw, big, legacy, a_dir, "v1") + meta_b = build_truth_layer(raw, big, + legacy + scanner_ranges_prefixes(csv), + b_dir, "v1") + assert meta_a.fingerprint != meta_b.fingerprint # 换版前提 + md = compose_monthly_md( + _readout(), [], [], [], [], + _header_meta(corpus={"truth": {"version": meta_b.version, + "fingerprint": meta_b.fingerprint}}, + prev_fingerprints={"truth": meta_a.fingerprint})) + reset = [ln for ln in md.splitlines() if "换版=趋势重置" in ln] + assert reset, "换版未出注记行" + assert "truth" in reset[0] and meta_a.fingerprint in reset[0] \ + and meta_b.fingerprint in reset[0] + + def test_md_empty_action_block_renders_placeholder(): md = compose_monthly_md(_readout(), [], [], [], [], _header_meta()) assert "OPEN 0 / CARRIED 0 / DONE 0" in md diff --git a/backend/tests/eval/test_eval_truth_layer.py b/backend/tests/eval/test_eval_truth_layer.py index c0c3684e..e56371b7 100644 --- a/backend/tests/eval/test_eval_truth_layer.py +++ b/backend/tests/eval/test_eval_truth_layer.py @@ -3,6 +3,7 @@ import pytest from ipdb._eval.assets import fingerprint, load_asset +from ipdb._eval.layer_builders import scanner_ranges_prefixes from ipdb._eval.truth_layer import ( TruthLayerDegradedError, apply_rir_filter, @@ -95,6 +96,40 @@ def test_build_happy_path_publishes_truth_asset(tmp_path): assert members == ["5.1.0.5", "9.9.9.9"] and loaded.kind == "truth" +def test_scanner_ranges_join_two_arm_exclusion_and_fingerprint(tmp_path): + # SR-T2 D3(#213 / ADR 0011): ops 两臂对照 —— arm A legacy scanlist, + # arm B legacy+自报段(D1 scanner_ranges_prefixes 供表);同 raw/rir + # 输入,验证装配侧剔除生效 + 语料版本随附(S12: 成员指纹与 + # scanlist_fingerprint 两臂皆变)。v6 前缀混排过 build_truth_layer + # 不炸(值域与 v4 永不重叠)。 + csv = tmp_path / "scanner_ranges.csv" + csv.write_text("40.124.175.0/24,Stretchoid\n2600:1f00::/24,Amazon\n", + encoding="utf-8") + legacy = ["10.0.0.0/8"] # 不覆盖 40.124.175.29(census 例证 IP) + raw = ["40.124.175.29", "5.1.0.5", "9.9.9.9"] + a_dir, b_dir = tmp_path / "a", tmp_path / "b" + meta_a = build_truth_layer(raw, _BIG_RANGES, legacy, a_dir, "v1") + meta_b = build_truth_layer(raw, _BIG_RANGES, + legacy + scanner_ranges_prefixes(csv), + b_dir, "v1") + mem_a, load_a = load_asset(a_dir) + mem_b, load_b = load_asset(b_dir) + # 40.124.175.29:arm A 在场(污染),arm B 剔除;段外成员两臂同存 + assert mem_a == ["40.124.175.29", "5.1.0.5", "9.9.9.9"] # 去重定序 + assert mem_b == ["5.1.0.5", "9.9.9.9"] + # removed_by_scanner_filter 差值精确(arm A legacy 不覆盖该 IP → 0) + assert meta_a.params["removed_by_scanner_filter"] == 0 + assert meta_b.params["removed_by_scanner_filter"] == 1 + # S12 换版随附:语料指纹与 scanlist 指纹两臂皆变 + assert meta_a.fingerprint != meta_b.fingerprint + assert meta_a.params["scanlist_fingerprint"] != \ + meta_b.params["scanlist_fingerprint"] + # 两臂 load_asset 往返一致(指纹闸: 证据永不灭失) + assert load_a.fingerprint == meta_a.fingerprint + assert load_b.fingerprint == meta_b.fingerprint + assert load_a.kind == load_b.kind == "truth" + + def test_parse_accepts_delegated_extended_8th_field(): """真 delegated-extended 带第 8 列 opaque-id(2026-10-14 首真数据实测 发现:7 字段断言 → 镜像五文件 0 段)。≥7 字段即收,富余列忽略。""" diff --git a/backend/tests/lookup/test_scanner_ranges_label.py b/backend/tests/lookup/test_scanner_ranges_label.py new file mode 100644 index 00000000..298a63d6 --- /dev/null +++ b/backend/tests/lookup/test_scanner_ranges_label.py @@ -0,0 +1,88 @@ +"""SR-T3 (#214): scanner_ranges 自报段标签 merge → API 契约钉测。 + +链路(spec /tmp/scanner-ranges-spec.md §设计定案 5;EXPLORE §3 2026-10-26): +真 ScannerRangesSource + 离线 fixture CSV(不碰网络)→ lookup() 全链(真 +to_observation / _assess_classification,仅 monkeypatch _sources,先例 +test_lookup_pipeline.py)→ ClassificationAssessment.details 的 +native_categories 键 → LookupResult.to_dict() → LookupResultOut 响应模型 +(ClassificationOut.details: list[dict] 通用透传,extra=allow)。产品代码 +零改(用户裁定零新 UI 机制;透传已通用),以测试钉住不漂移。 + +红证口径(本票零实现,测试的咬合性经 tamper-red 证明,不落 commit): +摘 _merge.py detail 组装的 native_categories 条件键 → 本文件必红;摘 +SourceDetailRow.tsx chip 渲染 → 前端 SourceDetailRow.test.tsx 必红。 + +verdict_conflict 不点亮(spec §5):scanner_ranges 无良性断言(verdict= +malicious 是行为类型断言,"aggressive scanning" 词表,拒 benign② 路线), +指控组内无 benign 对立 → 无冲突面,断言 False 钉住。 +""" +import pytest + +from ipdb._api_models import LookupResultOut +from ipdb._sources.scanner_ranges import ScannerRangesSource + +# spec §设计定案 6 验收冒烟例(离线版):40.124.175.29 = stretchoid 段内 +# (census 例证 IP)= malicious 判决 + scanner 证人行标签 +# 「研究扫描 · Stretchoid(自报段)」。fixture 喂该段即可,不必真抓。 +_FIXTURE = "40.124.175.0/24,Stretchoid\n" +_LABEL = "研究扫描 · Stretchoid(自报段)" + +# detail 期望形状:scanner_ranges 证据只带四键(源/r/verdict/标签),无 +# first_seen 等其他条件键 → 整 dict 精确钉(逐字段一键不多不少)。 +_EXPECTED_DETAIL = { + "source": "scanner_ranges", + "reliability": 0.90, + "verdict": "malicious", + "native_categories": [_LABEL], +} + + +@pytest.fixture +def scanner_src(tmp_path, monkeypatch): + """真 ScannerRangesSource(离线 fixture,rebuild 打开 reader 打查询门)。""" + (tmp_path / "scanner_ranges.csv").write_text(_FIXTURE) + src = ScannerRangesSource(data_dir=tmp_path) + assert src.rebuild() == 1 + import ipdb._registry as reg + monkeypatch.setattr(reg, "_sources", [src]) + return src + + +def test_acceptance_ip_malicious_verdict_with_label_detail(scanner_src): + """验收例:恶意判决照旧 + 指控证人照列 + 标签全文在 detail + + verdict_conflict 不点亮(无良性断言 → 无冲突面)。""" + from ipdb._registry import lookup + r = lookup("40.124.175.29") + + # 恶意判决照旧(spec §5):threat 摘要 verdict=malicious,scanner 在指控类型内 + # (typed LookupResult 无 threat 属性,摘要经 threat_summary() 序列化面取) + summary = r.threat_summary() + assert summary["verdict"] == "malicious" + assert "scanner" in summary["types"] + + ca = r.classifications["scanner"] + assert ca.type == "scanner" + assert ca.verdict == "malicious" + assert ca.detected is True + assert ca.corroborated is False # 单源自报段,无第二票(宁少算) + assert ca.verdict_conflict is False # 无良性断言 → 冲突面不存在 + assert ca.details == [_EXPECTED_DETAIL] + # 证人行全部为指控观测,无 benign(verdict=malicious 行为类型断言) + assert all(d["verdict"] == "malicious" for d in ca.details) + + +def test_label_survives_api_response_model(scanner_src): + """API 出海面:GET /api/lookup/{ip} 与 POST /api/query/stream 同构 + (LookupResultOut 响应模型);标签键经响应模型逐字段可见,形状零漂移。""" + from ipdb._registry import lookup + r = lookup("40.124.175.29") + + out = LookupResultOut.model_validate(r.to_dict()) + api_ca = out.classifications["scanner"] + assert api_ca.verdict == "malicious" + assert api_ca.verdict_conflict is False + assert api_ca.details == [_EXPECTED_DETAIL] + # 序列化后仍逐字段可见(response_model 出的真实 JSON 形状) + dumped = out.model_dump() + assert dumped["classifications"]["scanner"]["details"] == [_EXPECTED_DETAIL] + assert dumped["classifications"]["scanner"]["verdict_conflict"] is False diff --git a/backend/tests/sources/test_scanner_ranges.py b/backend/tests/sources/test_scanner_ranges.py new file mode 100644 index 00000000..ac35a628 --- /dev/null +++ b/backend/tests/sources/test_scanner_ranges.py @@ -0,0 +1,199 @@ +"""scanner_ranges (scanner vendors' self-declared ranges) — Evidence shape assertions. + +Covers: _FEEDS completeness (66 MISP warninglists + 1 Xpanse direct page, +name set == spec verbatim), MISP list.json parsing (v4/v6 mixed, bare-IP → +host route, garbage drops), Xpanse HTML table parsing (9 spec-verbatim CIDR ++ noise), harvest → scanner/malicious Evidence with the native label +「研究扫描 · <厂商/机构>(自报段)」on native_categories (f3csystems.py:53 +先例; spec 2026-10-26 补定 — native_types 只喂 ASSET_SLOTS, scanner 无槽; +渲染链 _merge detail dict → _api_models.py:56 → 前端 SourceDetailRow chip), +dual-family routing, rebuild-level dedup (identical evidence collapses, +distinct vendors coexist), health contract, and registry metadata +(category=threat, r=0.90, authoritative_for=() — corroboration only). +Offline: fixtures only, no network. +""" +import json + +import pytest + +from ipdb._evidence import Evidence +from ipdb._sources.scanner_ranges import ( + _FEEDS, + _MISP_URL_TMPL, + _XPANSE_URL, + ScannerRangesSource, + _parse, +) + +# Spec-verbatim 66 MISP warninglist names (census 2026-10-26, CC0/CIRCL, +# -nt- lists deliberately excluded — observation-extracted, not self-declared). +SPEC_MISP_NAMES = frozenset( + """a10networks-scanning academyforinternetresearch-scanning adscore-scanning + alphastrike-scanning binaryedge-scanning bitsight-scanning + bufferoverrun-scanning censys-scanning circl-security-scanning + circl-ssh-scanning criminalip-scanning crowdstrikefalconsurface-scanning + cybcube-scanning cybergreen-scanning cyberok-scanning + cyberresilience-scanning cypex-scanning dataplane-scanning + deepfield-scanning detectify-scanning edgewatch-scanning eset-scanning + fofa-scanning groupib-scanning hackertarget-scanning hadrian-scanning + hunter-scanning infrawatch-scanning internetcleanup-scanner + internetmeasurementresearch-scanning internetmetrics-scanning + internetresearchproject-scanning internettl-scanning intrinsec-scanning + intruder-scanning ipinfo-scanning ipip-scanning ki3-scanning knoq-scanning + leakix-scanning mandiantasm-scanning modat-scanner netlas-scanning + netscout-scanning netsecscan-scanning odin-scanning onlyscans-scanning + onyphe-scanner pocnroll-scanning probethenet-scanning quake-scanning + rapid7-scanning recordedfuture-scanning recyber-scanning + researchscanner-scanning rootevidence-scanning securityresearch-scanning + shadowforce-scanning shodan-scanning stretchoid-scanning + tenableasm-scanning university-scanning visionheight-scanning + xmco-scanning zern-scanning zoomeye-scanning""".split() +) + +# Spec-verbatim Xpanse direct page CIDR (cortex-docs scanning-activity). +XPANSE_CIDRS = [ + "144.86.173.0/24", "147.185.132.0/23", "162.216.149.0/24", + "162.216.150.0/24", "172.105.147.0/24", "198.235.24.0/24", + "205.210.31.0/24", "216.25.88.0/21", "35.203.210.0/23", +] + +# Combined-intermediate fixture (cidr,vendor) — exactly what download() writes. +_FIXTURE = """\ +162.142.125.0/24,Censys +167.248.133.0/24,Censys +2a0e:7b84::/30,Alpha Strike Labs +40.124.175.0/24,Stretchoid +198.235.24.0/24,Palo Alto Xpanse +""" + + +def test_feeds_complete_66_misp_plus_1_xpanse(): + """66+1 completeness: name set == spec verbatim; MISP rows share one + raw.githubusercontent URL template; Xpanse row is the direct docs page.""" + assert len(_FEEDS) == 67 + names = {name for name, _url, _vendor, _fmt in _FEEDS} + assert names == SPEC_MISP_NAMES | {"xpanse"} + misp = [f for f in _FEEDS if f[3] == "misp"] + assert len(misp) == 66 + for name, url, _vendor, _fmt in misp: + assert url == _MISP_URL_TMPL.format(name=name) + xpanse = [f for f in _FEEDS if f[0] == "xpanse"][0] + assert xpanse[1] == _XPANSE_URL + assert xpanse[2] == "Palo Alto Xpanse" + # 红线锚:OpenFilters 镜像/-nt-/GreyNoise 一概不接(spec 定案)。 + assert not any("openfilters" in f[1] for f in _FEEDS) + assert not any("-nt-" in f[0] for f in _FEEDS) + assert not any("greynoise" in f[1].lower() for f in _FEEDS) # API-only 红线外(spec 定案 1;Spec 终审 P2 补齐) + + +def test_parse_misp_list_json_dual_family_and_garbage(): + """实测缓存形态(/tmp/misl_censys-scanning.json):list 数组 v4/v6 混排, + 裸 IP 收作 /32(实测 66 列表含 22 条裸 IP),非字符串/坏条目弃。""" + data = json.dumps({ + "name": "Censys IP Ranges Used for Scanning", + "description": "List containing IP's associated with Censys scanning", + "version": 3, + "list": [ + "162.142.125.0/24", # v4 CIDR + "2602:801:f000::/44", # v6 混排 + "162.142.124.7", # 裸 IP → /32 + "not-an-ip", # 坏条目 → 弃 + 12345, # 非字符串 → 弃 + "", # 空串 → 弃 + ], + }).encode() + assert list(_parse(data, "misp")) == [ + "162.142.125.0/24", "2602:801:f000::/44", "162.142.124.7/32", + ] + + +def test_parse_xpanse_html_nine_cidr_verbatim(): + """Xpanse 官方页直读:表格文本里的 9 CIDR 原样提出,页内噪声 + (裸 IP/版本号/邮箱)不产 CIDR。""" + html = """ + +

Cortex Xpanse Scanning Activity

+

Xpanse performs internet-wide scanning from the ranges below.

+ + + + + + + + + + +
35.203.210.0/23
144.86.173.0/24
147.185.132.0/23
162.216.149.0/24
162.216.150.0/24
172.105.147.0/24
198.235.24.0/24
205.210.31.0/24
216.25.88.0/21
+
Contact scans@paloaltonetworks.example · page v1.2 · 2026-10-26
+ +""".encode() + assert sorted(_parse(html, "xpanse")) == sorted(XPANSE_CIDRS) + + +def test_parse_xpanse_zero_cidr_raises(): + """零命中守卫:页面改版提不出任何 CIDR 时炸响,拒绝排除面 + 无声缩水(与 loader 缺 CSV raise 同律;Spec 终审 P2)。""" + with pytest.raises(ValueError, match="0 CIDR"): + list(_parse(b"docs redesign, no ranges here", "xpanse")) + + +def test_scanner_ranges_loads_and_routes(tmp_path): + (tmp_path / "scanner_ranges.csv").write_text(_FIXTURE) + s = ScannerRangesSource(data_dir=tmp_path) + assert s.rebuild() == 4 # rebuild() 契约 = v4 族键数(4 v4 + 1 v6) + # Stretchoid 自报段内 IP(spec 验收烟囱例 40.124.175.29)→ 证据契约整形状 + rec = s.query("40.124.175.29")[0] + assert rec == Evidence( + classification_type="scanner", + verdict="malicious", + reliability=0.90, + native_categories=["研究扫描 · Stretchoid(自报段)"], + ).to_dict() + # Censys / Xpanse 标签字面量(native_categories 路线,spec 补定) + assert s.query("162.142.125.10")[0]["native_categories"] == [ + "研究扫描 · Censys(自报段)"] + assert s.query("198.235.24.1")[0]["native_categories"] == [ + "研究扫描 · Palo Alto Xpanse(自报段)"] + # 双族:v6 走并行族 reader + assert s.query("2a0e:7b84::1")[0]["native_categories"] == [ + "研究扫描 · Alpha Strike Labs(自报段)"] + # 段外 miss + assert s.query("8.8.8.8") == {} + + +def test_dedup_identical_collapses_distinct_vendor_keeps(tmp_path): + (tmp_path / "scanner_ranges.csv").write_text( + "10.1.0.0/16,Censys\n" + "10.1.0.0/16,Censys\n" # 完全同证据 → rebuild 层 acc 去重 + "10.1.0.0/16,Shodan\n" # 同 CIDR 异厂 → 两票并存 + ) + s = ScannerRangesSource(data_dir=tmp_path) + assert s.rebuild() == 1 + recs = s.query("10.1.0.5") + assert len(recs) == 2 + labels = sorted(r["native_categories"][0] for r in recs) + assert labels == ["研究扫描 · Censys(自报段)", "研究扫描 · Shodan(自报段)"] + + +def test_health_loaded_not_stale(tmp_path): + (tmp_path / "scanner_ranges.csv").write_text(_FIXTURE) + s = ScannerRangesSource(data_dir=tmp_path) + s.rebuild() + h = s.health() + assert h.loaded is True + assert h.record_count == 4 # DQ-2:record_count = env 主库(v4 族)键数 + assert h.covered_v6_nets == 1 # v6 并行族键数 + assert h.is_stale is False # stale_days=7; fixture just written + + +def test_registry_metadata_threat_corroboration_only(): + """自动发现注册(_discover_sources):category/reliability/authoritative_for + 落册形态;authoritative_for=() 保证不进 AUTHORITATIVE_SOURCES(SM-F1, + test_source_decls 的权威矩阵精确断言不被扰动)。""" + import ipdb._registry as reg + src = next(s for s in reg._sources if s.name == "scanner_ranges") + assert src.category == "threat" + assert src.reliability == 0.90 + assert src.authoritative_for == () + assert src.stale_days == 7 diff --git a/frontend/src/components/__tests__/SourceDetailRow.test.tsx b/frontend/src/components/__tests__/SourceDetailRow.test.tsx index bc92e988..5c7473f7 100644 --- a/frontend/src/components/__tests__/SourceDetailRow.test.tsx +++ b/frontend/src/components/__tests__/SourceDetailRow.test.tsx @@ -86,6 +86,22 @@ describe("SourceDetailRow", () => { expect(screen.getByText(/\[16\]/)).toBeInTheDocument(); }); + it("renders the full scanner self-declared-range label chip (SR-T3 #214)", () => { + // spec §5:scanner 证人行 native 标签「研究扫描 · <厂商/机构>(自报段)」 + // 全文原样出 chip(通用 native_categories 渲染,零新 UI 机制;钉法照上例)。 + // 验收例 = Stretchoid(40.124.175.29):标签全文一字不差地可见。 + const d: ClassificationDetail = { + source: "scanner_ranges", + verdict: "malicious", + reliability: 0.9, + native_categories: ["研究扫描 · Stretchoid(自报段)"], + }; + renderWithI18n(); + expect( + screen.getByText("[研究扫描 · Stretchoid(自报段)]") + ).toBeInTheDocument(); + }); + it("ignores retired extra.native_type (fallback removed)", () => { const d: ClassificationDetail = { ...base, extra: { native_type: "PUB" } }; renderWithI18n();