From 3db02c99000740212706c0a0a504a6fed33a1e07 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 2 Oct 2026 15:11:16 +0900 Subject: [PATCH 01/22] fix(dnsbl): split TXT metadata into wire-sized character strings --- crates/waf-ids-core/src/lib.rs | 69 ++++++++++++------ crates/waf-ids-core/tests/dnsbl_txt_chunks.rs | 68 ++++++++++++++++++ crates/waf-ids-core/tests/fuzz_invariants.rs | 34 ++++----- .../waf-ids-core/tests/support/dnsbl_txt.rs | 64 +++++++++++++++++ docs/doctoring/dnsbl-txt-character-strings.md | 69 ++++++++++++++++++ docs/fuzzing.md | 2 +- fuzz/fuzz_targets/fuzz_dnsbl_zone.rs | 58 +++++++-------- fuzz/support/dnsbl_txt.rs | 6 ++ tests/dnsbl_txt_export.rs | 71 +++++++++++++++++++ tests/support/dnsbl_txt.rs | 64 +++++++++++++++++ 10 files changed, 426 insertions(+), 79 deletions(-) create mode 100644 crates/waf-ids-core/tests/dnsbl_txt_chunks.rs create mode 100644 crates/waf-ids-core/tests/support/dnsbl_txt.rs create mode 100644 docs/doctoring/dnsbl-txt-character-strings.md create mode 100644 fuzz/support/dnsbl_txt.rs create mode 100644 tests/dnsbl_txt_export.rs create mode 100644 tests/support/dnsbl_txt.rs diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index f9673e0c..015715f2 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -1393,6 +1393,9 @@ pub fn readiness_check(id: &str, passed: bool, evidence: &str) -> ReadinessCheck } } +/// Export IPv4 DNSBL entries with loopback answers and lossless TXT metadata. +/// Each TXT character string is at most 255 decoded UTF-8 bytes (RFC 1035 +/// sections 3.3 and 3.3.14); longer metadata uses adjacent quoted strings. pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { let mut out = format!("$ORIGIN {}.\n$TTL 300\n", sanitize_zone_origin(origin)); for entry in entries { @@ -1446,9 +1449,18 @@ pub fn reverse_ipv4_for_dnsbl(octets: [u8; 4]) -> String { format!("{}.{}.{}.{}", octets[3], octets[2], octets[1], octets[0]) } +/// Escape metadata and split it at UTF-8 character boundaries into adjacent +/// TXT strings. Count decoded bytes, not master-file escape characters, so +/// every string fits RFC 1035's one-octet length without dropping metadata. fn escape_txt(value: &str) -> String { let mut out = String::with_capacity(value.len()); + let mut decoded_bytes = 0; for ch in value.chars() { + if decoded_bytes + ch.len_utf8() > 255 { + out.push_str("\" \""); + decoded_bytes = 0; + } + decoded_bytes += ch.len_utf8(); match ch { '\\' => out.push_str("\\\\"), '"' => out.push_str("\\\""), @@ -1468,6 +1480,10 @@ fn escape_txt(value: &str) -> String { out } +#[cfg(test)] +#[path = "../tests/support/dnsbl_txt.rs"] +mod dnsbl_txt; + #[cfg(test)] mod tests { use super::*; @@ -1553,29 +1569,38 @@ mod tests { assert!(scored.score >= BLOCK_SCORE); } - /// Assert every double quote inside a TXT payload is backslash-escaped, i.e. - /// preceded by an odd run of backslashes. Mirrors the fuzz/proptest invariant - /// so regressions in zone escaping fail as a plain unit test too. + /// Check quoted-string grammar and byte limits with the same independent + /// decoder used by the stable property tests and coverage-guided fuzzing. fn assert_txt_quotes_escaped(zone: &str) { - for line in zone.lines().filter(|l| l.contains(" IN TXT ")) { - let start = line.find('"').expect("TXT record has an opening quote"); - let end = line.rfind('"').expect("TXT record has a closing quote"); - let payload = &line.as_bytes()[start + 1..end]; - for (idx, &b) in payload.iter().enumerate() { - if b == b'"' { - let mut backslashes = 0usize; - let mut j = idx; - while j > 0 && payload[j - 1] == b'\\' { - backslashes += 1; - j -= 1; - } - assert!( - backslashes % 2 == 1, - "unescaped quote in TXT payload: {line:?}" - ); - } - } - } + crate::dnsbl_txt::assert_zone_txt_valid(zone); + } + + #[test] + fn escape_long_txt_uses_valid_adjacent_strings() { + let value = format!("{}ํ•œ๐Ÿ›ก\\\"\n", "x".repeat(254)); + let text = format!("\"{}\"", escape_txt(&value)); + let strings = crate::dnsbl_txt::decode_txt_rdata(&text); + assert_eq!(strings[0].len(), 254); + assert_eq!(strings.concat(), value.as_bytes()); + let ipv6 = DnsblEntry { + address: "2001:db8::1".parse().unwrap(), + code: "127.0.0.2".to_string(), + reason: value, + source: "unit".to_string(), + ttl_seconds: 300, + prefix_len: None, + }; + let zone = export_dnsbl_zone("dnsbl.example", &[ipv6]); + assert_eq!(zone, "$ORIGIN dnsbl.example.\n$TTL 300\n"); + } + + #[test] + fn escape_empty_txt_preserves_the_empty_character_string() { + assert_eq!(escape_txt(""), ""); + assert_eq!( + crate::dnsbl_txt::decode_txt_rdata("\"\""), + [Vec::::new()] + ); } #[test] diff --git a/crates/waf-ids-core/tests/dnsbl_txt_chunks.rs b/crates/waf-ids-core/tests/dnsbl_txt_chunks.rs new file mode 100644 index 00000000..63936f58 --- /dev/null +++ b/crates/waf-ids-core/tests/dnsbl_txt_chunks.rs @@ -0,0 +1,68 @@ +//! DNSBL publishing regressions for RFC 1035 section 3.3 character strings. + +#[path = "support/dnsbl_txt.rs"] +mod dnsbl_txt; + +use waf_ids_core::{DnsblEntry, export_dnsbl_zone, validate_dnsbl}; + +/// Construct a valid admission fixture without changing DNSBL classification. +fn entry(reason: String, source: String) -> DnsblEntry { + DnsblEntry { + address: "192.0.2.10".parse().unwrap(), + code: "127.0.0.2".to_string(), + reason, + source, + ttl_seconds: 300, + prefix_len: None, + } +} + +/// Assert independent TXT decoding preserves the admitted reason/source bytes. +fn assert_round_trip(entry: DnsblEntry) -> Vec> { + validate_dnsbl(&entry).unwrap(); + let expected = format!("{} source={}", entry.reason, entry.source); + let zone = export_dnsbl_zone("dnsbl.example", &[entry]); + assert_eq!(zone.lines().count(), 4); + assert!(zone.contains("10.2.0.192 IN A 127.0.0.2\n")); + dnsbl_txt::assert_zone_txt_valid(&zone); + let text = zone + .lines() + .find_map(|l| l.split_once(" IN TXT ").map(|(_, t)| t)) + .unwrap(); + let strings = dnsbl_txt::decode_txt_rdata(text); + assert_eq!(strings.concat(), expected.as_bytes()); + strings +} + +#[test] +fn txt_chunking_counts_decoded_bytes_at_the_255_byte_boundary() { + for total in [254, 255, 256, 510, 511] { + let strings = assert_round_trip(entry("x".repeat(total - 12), "unit".to_string())); + assert_eq!(strings.len(), total.div_ceil(255)); + } + let strings = assert_round_trip(entry("\\\"".repeat(100), "unit".to_string())); + assert_eq!( + strings.len(), + 1, + "escaped syntax is not decoded payload size" + ); +} + +#[test] +fn txt_chunking_keeps_multibyte_utf8_and_escapes_lossless() { + let reason = format!("{}ํ•œ๐Ÿ›ก\\\"\n\r\t\u{0}\u{85}", "x".repeat(254)); + let strings = assert_round_trip(entry(reason, "์†Œ์Šค\\\"\n".repeat(90))); + assert_eq!(strings[0].len(), 254); + for string in strings { + std::str::from_utf8(&string).expect("split must not bisect UTF-8"); + } +} + +#[test] +fn long_admitted_txt_metadata_is_losslessly_split_into_wire_sized_strings() { + let strings = assert_round_trip(entry("x".repeat(600), "unit".to_string())); + assert_eq!( + strings.iter().map(Vec::len).collect::>(), + [255, 255, 102] + ); +} diff --git a/crates/waf-ids-core/tests/fuzz_invariants.rs b/crates/waf-ids-core/tests/fuzz_invariants.rs index d2d75320..f55f0a00 100644 --- a/crates/waf-ids-core/tests/fuzz_invariants.rs +++ b/crates/waf-ids-core/tests/fuzz_invariants.rs @@ -6,6 +6,9 @@ //! without a nightly toolchain. The fuzz targets explore far deeper; these keep //! a fast, always-green signal. +#[path = "support/dnsbl_txt.rs"] +mod dnsbl_txt; + use proptest::prelude::*; use std::net::{IpAddr, Ipv4Addr}; use waf_ids_core::{ @@ -135,28 +138,15 @@ proptest! { } } - for line in zone.lines() { - if !line.contains(" IN TXT ") { - continue; - } - let (Some(start), Some(end)) = (line.find('"'), line.rfind('"')) else { - continue; - }; - if end <= start { - continue; - } - let payload = &line.as_bytes()[start + 1..end]; - for (idx, &b) in payload.iter().enumerate() { - if b == b'"' { - let mut backslashes = 0usize; - let mut j = idx; - while j > 0 && payload[j - 1] == b'\\' { - backslashes += 1; - j -= 1; - } - prop_assert!(backslashes % 2 == 1, "unescaped quote in TXT payload"); - } - } + dnsbl_txt::assert_zone_txt_valid(&zone); + let txt_lines: Vec<_> = zone.lines().filter_map(|l| l.split_once(" IN TXT ")).collect(); + let expected: Vec<_> = entries.iter().filter(|e| { + matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127) + }).collect(); + prop_assert_eq!(txt_lines.len(), expected.len()); + for ((_, text), entry) in txt_lines.iter().zip(expected) { + let decoded = dnsbl_txt::decode_txt_rdata(text).concat(); + prop_assert_eq!(decoded, format!("{} source={}", entry.reason, entry.source).into_bytes()); } } } diff --git a/crates/waf-ids-core/tests/support/dnsbl_txt.rs b/crates/waf-ids-core/tests/support/dnsbl_txt.rs new file mode 100644 index 00000000..ac6bbd6a --- /dev/null +++ b/crates/waf-ids-core/tests/support/dnsbl_txt.rs @@ -0,0 +1,64 @@ +//! Independent master-file TXT decoder for unit/property/fuzz assertions. + +/// Decode only the quoted-string grammar emitted by Wardnet, rejecting malformed +/// escapes, unquoted text and character strings exceeding RFC 1035's byte limit. +/// This deliberately does not call the production encoder or share its limits. +pub fn decode_txt_rdata(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut offset = 0; + let mut strings = Vec::new(); + while offset < bytes.len() { + assert_eq!(bytes[offset], b'"', "TXT string must begin with a quote"); + offset += 1; + let mut decoded = Vec::new(); + loop { + let byte = *bytes.get(offset).expect("unterminated TXT string"); + offset += 1; + match byte { + b'"' => break, + b'\\' => { + let escaped = *bytes.get(offset).expect("unterminated TXT escape"); + if escaped.is_ascii_digit() { + let digits = bytes.get(offset..offset + 3).expect("short decimal escape"); + assert!(digits.iter().all(u8::is_ascii_digit)); + let value = digits + .iter() + .fold(0u16, |v, d| v * 10 + u16::from(d - b'0')); + decoded.push(u8::try_from(value).expect("decimal escape exceeds one byte")); + offset += 3; + } else { + assert!(matches!(escaped, b'"' | b'\\'), "unexpected TXT escape"); + decoded.push(escaped); + offset += 1; + } + } + b'\n' | b'\r' => panic!("raw line break in TXT string"), + byte => decoded.push(byte), + } + } + assert!( + decoded.len() <= 255, + "TXT character string exceeds 255 bytes: {}", + decoded.len() + ); + strings.push(decoded); + if offset < bytes.len() { + assert_eq!(bytes[offset], b' ', "TXT strings must be separated"); + offset += 1; + assert!(offset < bytes.len(), "trailing TXT separator"); + } + } + assert!(!strings.is_empty(), "TXT requires a character string"); + strings +} + +/// Check every TXT record's grammar and wire-size limits, retaining all bytes +/// for callers that also assert lossless reason/source round trips. +pub fn assert_zone_txt_valid(zone: &str) { + for line in zone.lines() { + if let Some((_, text)) = line.split_once(" IN TXT ") { + let strings = decode_txt_rdata(text); + assert!(strings.iter().all(|string| string.len() <= 255)); + } + } +} diff --git a/docs/doctoring/dnsbl-txt-character-strings.md b/docs/doctoring/dnsbl-txt-character-strings.md new file mode 100644 index 00000000..9703004b --- /dev/null +++ b/docs/doctoring/dnsbl-txt-character-strings.md @@ -0,0 +1,69 @@ +# DNSBL TXT character-string publishing boundary + +## Finding and owned repair + +At protected Wardnet `f8260f1e03836039ff9463dd99fa982e4e270c4b`, +`validate_dnsbl` accepts nonempty reasons and sources without a per-string +length restriction, but `export_dnsbl_zone` encloses their combined metadata in +one TXT character string. A valid entry with 600 ASCII reason bytes and source +`unit` generates a 612-byte string that dnspython 2.8.0 rejects as `string too +long`. A 255-byte positive control is accepted by the same real parser. The +primary DNS Rdata API is documented by the parser project.[2] + +This slice belongs to Wardnet's DNSBL publishing domain, not feed snapshot +ownership, MISP translation, egress, identity, deployment or source acquisition. +Existing PR #167 retains the MISP and shared DNSBL ownership fixes; it is not +copied or replaced. The other open-PR export-source comparisons found no TXT +chunking repair; the official-source preservation PR #115 changes CIDR export +selection and is left untouched. + +## Protocol contract + +RFC 1035 section 3.3 defines a character string with a one-octet length and at +most 255 data octets. Section 3.3.14 permits one or more character strings in +TXT RDATA.[1] The exporter now splits the combined reason/source metadata at +UTF-8 character boundaries before exceeding 255 decoded bytes. Escaped quotes, +backslashes and decimal control-byte syntax count as their decoded bytes, not +the number of master-file characters. Short output stays byte-for-byte +compatible; long output uses adjacent quoted strings in the same TXT record. + +No metadata is truncated. Admission, stored fields, address/code selection, +feed ownership and the HTTP API shape remain unchanged. This is not complete +DNS zone validation: aggregate RDATA size, origin/name syntax limits, IPv6/CIDR +publication and authoritative SOA/NS provisioning remain separate concerns. +In particular, this repair does not assert that arbitrarily large total TXT +RDATA fits the protocol's two-octet RDLENGTH. + +## Executed regression chain + +- Before any production edit, `dnsbl_txt_chunks` failed with exit 101 on + `TXT character string exceeds 255 bytes: 612`. +- The same regression passed after a byte-counted UTF-8-safe split was added to + the existing escape function. +- Stable tests cover 254/255/256/510/511-byte boundaries, long metadata, + multibyte UTF-8, quote/backslash/control escapes and short-output compatibility. +- An HTTP regression drives the real router from authenticated DNSBL admission + to `/dnsbl/zone`, checking metadata bytes and absence of forged record lines. +- Extracted crate testing exposed a candidate packaging regression: the shared + test decoder initially lived outside the core crate. Its implementation now + resides in `crates/waf-ids-core/tests/support/`, with a test-only wrapper for + the separate fuzz workspace. The root HTTP test carries the same independent + decoder in its own `tests/support/` so each package includes its test source + without exporting a production API or adding a production shared kernel. +- Unit, stable proptest and the libFuzzer target use a test-only independent + quoted-string decoder. Chunk delimiters are parsed as grammar, rather than + mistakenly treating all interior quotes as injection. The decoder does not + call the production encoder or share its limit constant. +- An isolated Rust harness compiled the exact protected source and the repaired + source separately; real dnspython master-file parsing and TXT wire encoding + rejected the baseline 256/612-byte and UTF-8 fixtures and accepted every + repaired fixture with lossless bytes. No DNS server or deployment was changed. + +Full workspace gates, exact coverage and independent review are recorded in +the execution ledger. These regressions are protocol fixtures, not business +acceptance, detection-quality measurements, rendered UX or performance evidence. + +## Sources + +[1] https://www.rfc-editor.org/rfc/rfc1035.txt โ€” Mockapetris (1987), RFC 1035, sections 3.3 and 3.3.14 +[2] https://dnspython.readthedocs.io/en/latest/rdata.html โ€” Dnspython DNS Rdata API diff --git a/docs/fuzzing.md b/docs/fuzzing.md index b252299c..42c87838 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -19,7 +19,7 @@ entry points for arbitrary input. | `fuzz_score_request` | `waf_ids_core::score_request` | no panic on arbitrary path/query/body/IP; `reason` never empty; scoring deterministic | | `fuzz_appdata_json` | `serde_json::from_str::` (state file) | no panic; parsed values round-trip through serde | | `fuzz_parse_admin_tokens` | `waf_ids_ai_soc::parse_admin_tokens` | no panic; no empty token key; no empty principal actor value | -| `fuzz_dnsbl_zone` | `waf_ids_core::export_dnsbl_zone` / `validate_dnsbl` | no panic; every TXT payload fully escaped (no zone break-out); every published A-record response code is an IPv4 loopback literal (127.0.0.0/8) | +| `fuzz_dnsbl_zone` | `waf_ids_core::export_dnsbl_zone` / `validate_dnsbl` | no panic; quoted TXT strings decode losslessly, remain injection-safe and contain at most 255 decoded bytes each (RFC 1035 ยง3.3/ยง3.3.14); every published A-record response code is an IPv4 loopback literal (127.0.0.0/8) | ## Layout diff --git a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs index 4cf0937f..e310efa5 100644 --- a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs +++ b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs @@ -7,14 +7,17 @@ //! strings flow into the generated zone. Generation must never panic, and //! `validate_dnsbl` must never panic while classifying arbitrary entries. //! -//! Invariant: every TXT record payload is fully escaped โ€” inside the quoted -//! payload every `"` is backslash-escaped, so the zone can never be broken out -//! of by adversarial reason/source strings. +//! Invariant: every TXT record consists of valid escaped quoted strings with +//! at most 255 decoded bytes each; concatenation preserves reason/source bytes. +//! Quotes, backslashes and control characters cannot break out into zone lines. + +#[path = "../support/dnsbl_txt.rs"] +mod dnsbl_txt; use arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use waf_ids_core::{export_dnsbl_zone, validate_dnsbl, DnsblEntry}; +use waf_ids_core::{DnsblEntry, export_dnsbl_zone, validate_dnsbl}; /// A response code drawn from the raw fuzz bytes: arbitrary strings plus real IP /// literals (loopback, non-loopback IPv4, IPv6) so the zone A-record invariant @@ -110,35 +113,22 @@ fuzz_target!(|input: Input| { } } - // Every TXT record payload must be properly escaped: inside the wrapping - // quotes, each `"` must be backslash-escaped. Extract the payload between - // the first and last quote of each TXT line and verify no *unescaped* quote - // survives (a quote is escaped iff preceded by an odd run of backslashes). - for line in zone.lines() { - if !line.contains(" IN TXT ") { - continue; - } - let first = line.find('"'); - let last = line.rfind('"'); - if let (Some(start), Some(end)) = (first, last) { - if end <= start { - continue; - } - let bytes = &line.as_bytes()[start + 1..end]; - for (idx, &b) in bytes.iter().enumerate() { - if b == b'"' { - let mut backslashes = 0usize; - let mut j = idx; - while j > 0 && bytes[j - 1] == b'\\' { - backslashes += 1; - j -= 1; - } - assert!( - backslashes % 2 == 1, - "unescaped quote in TXT payload: {line:?}" - ); - } - } - } + // Check every string, including chunk separators, with an independent + // decoder; require both wire-size legality and lossless metadata bytes. + dnsbl_txt::assert_zone_txt_valid(&zone); + let txt_lines: Vec<_> = zone + .lines() + .filter_map(|l| l.split_once(" IN TXT ")) + .collect(); + let expected: Vec<_> = entries + .iter() + .filter(|e| matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127)) + .collect(); + assert_eq!(txt_lines.len(), expected.len()); + for ((_, text), entry) in txt_lines.iter().zip(expected) { + assert_eq!( + dnsbl_txt::decode_txt_rdata(text).concat(), + format!("{} source={}", entry.reason, entry.source).into_bytes() + ); } }); diff --git a/fuzz/support/dnsbl_txt.rs b/fuzz/support/dnsbl_txt.rs new file mode 100644 index 00000000..a579cb6f --- /dev/null +++ b/fuzz/support/dnsbl_txt.rs @@ -0,0 +1,6 @@ +//! Reuse the crate-local test oracle without exporting a production API. + +#[path = "../../crates/waf-ids-core/tests/support/dnsbl_txt.rs"] +mod oracle; + +pub use oracle::{assert_zone_txt_valid, decode_txt_rdata}; diff --git a/tests/dnsbl_txt_export.rs b/tests/dnsbl_txt_export.rs new file mode 100644 index 00000000..f43ab59e --- /dev/null +++ b/tests/dnsbl_txt_export.rs @@ -0,0 +1,71 @@ +//! HTTP admission-to-DNSBL-publishing regression without a live server. + +#[path = "support/dnsbl_txt.rs"] +mod dnsbl_txt; + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +#[tokio::test] +async fn admitted_long_dnsbl_metadata_exports_as_lossless_valid_txt() { + let app = build_app(AppState::seeded(Some("dnsbl-fixture-token".to_string()))); + let reason = format!("{}ํ•œ๐Ÿ›ก\\\"\n", "x".repeat(254)); + let source = "test:source\\\"\n".repeat(50); + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/dnsbl") + .header("content-type", "application/json") + .header("x-admin-token", "dnsbl-fixture-token") + .body(Body::from( + serde_json::json!({ + "address": "192.0.2.10", "code": "127.0.0.2", + "reason": reason, "source": source, "ttl_seconds": 300 + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let response = app + .oneshot( + Request::builder() + .uri("/dnsbl/zone") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()["content-type"], + "text/plain; charset=utf-8" + ); + let zone = String::from_utf8( + to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() + .to_vec(), + ) + .unwrap(); + dnsbl_txt::assert_zone_txt_valid(&zone); + assert_eq!(zone.lines().count(), 6, "seed plus admitted entry only"); + let text = zone + .lines() + .find_map(|line| line.strip_prefix("10.2.0.192 IN TXT ")) + .unwrap(); + let strings = dnsbl_txt::decode_txt_rdata(text); + assert!(strings.len() > 1); + assert_eq!( + strings.concat(), + format!("{reason} source={source}").as_bytes() + ); +} diff --git a/tests/support/dnsbl_txt.rs b/tests/support/dnsbl_txt.rs new file mode 100644 index 00000000..ac6bbd6a --- /dev/null +++ b/tests/support/dnsbl_txt.rs @@ -0,0 +1,64 @@ +//! Independent master-file TXT decoder for unit/property/fuzz assertions. + +/// Decode only the quoted-string grammar emitted by Wardnet, rejecting malformed +/// escapes, unquoted text and character strings exceeding RFC 1035's byte limit. +/// This deliberately does not call the production encoder or share its limits. +pub fn decode_txt_rdata(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut offset = 0; + let mut strings = Vec::new(); + while offset < bytes.len() { + assert_eq!(bytes[offset], b'"', "TXT string must begin with a quote"); + offset += 1; + let mut decoded = Vec::new(); + loop { + let byte = *bytes.get(offset).expect("unterminated TXT string"); + offset += 1; + match byte { + b'"' => break, + b'\\' => { + let escaped = *bytes.get(offset).expect("unterminated TXT escape"); + if escaped.is_ascii_digit() { + let digits = bytes.get(offset..offset + 3).expect("short decimal escape"); + assert!(digits.iter().all(u8::is_ascii_digit)); + let value = digits + .iter() + .fold(0u16, |v, d| v * 10 + u16::from(d - b'0')); + decoded.push(u8::try_from(value).expect("decimal escape exceeds one byte")); + offset += 3; + } else { + assert!(matches!(escaped, b'"' | b'\\'), "unexpected TXT escape"); + decoded.push(escaped); + offset += 1; + } + } + b'\n' | b'\r' => panic!("raw line break in TXT string"), + byte => decoded.push(byte), + } + } + assert!( + decoded.len() <= 255, + "TXT character string exceeds 255 bytes: {}", + decoded.len() + ); + strings.push(decoded); + if offset < bytes.len() { + assert_eq!(bytes[offset], b' ', "TXT strings must be separated"); + offset += 1; + assert!(offset < bytes.len(), "trailing TXT separator"); + } + } + assert!(!strings.is_empty(), "TXT requires a character string"); + strings +} + +/// Check every TXT record's grammar and wire-size limits, retaining all bytes +/// for callers that also assert lossless reason/source round trips. +pub fn assert_zone_txt_valid(zone: &str) { + for line in zone.lines() { + if let Some((_, text)) = line.split_once(" IN TXT ") { + let strings = decode_txt_rdata(text); + assert!(strings.iter().all(|string| string.len() <= 255)); + } + } +} From f711e012c8f87a9097382647238945606e76d696 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 00:05:06 +0900 Subject: [PATCH 02/22] fix(dnsbl): preserve bounded cache lifetimes in published RRsets --- crates/waf-ids-core/src/lib.rs | 115 ++++++++++++++++++- crates/waf-ids-core/tests/dnsbl_ttl.rs | 103 +++++++++++++++++ crates/waf-ids-core/tests/fuzz_invariants.rs | 33 +++--- docs/doctoring/dnsbl-cache-lifetimes.md | 48 ++++++++ docs/fuzzing.md | 2 +- fuzz/fuzz_targets/fuzz_dnsbl_zone.rs | 30 +++-- scripts/smoke.sh | 3 +- tests/dnsbl_ttl_export.rs | 100 ++++++++++++++++ 8 files changed, 394 insertions(+), 40 deletions(-) create mode 100644 crates/waf-ids-core/tests/dnsbl_ttl.rs create mode 100644 docs/doctoring/dnsbl-cache-lifetimes.md create mode 100644 tests/dnsbl_ttl_export.rs diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index 015715f2..8198395b 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -5,6 +5,9 @@ use std::str::FromStr; use std::time::{SystemTime, UNIX_EPOCH}; pub const BLOCK_SCORE: u16 = 50; +// RFC 2181 section 8 permits TTLs from zero through 2^31 - 1; Wardnet +// separately requires a nonzero lifetime at DNSBL admission. +const DNSBL_MAX_TTL_SECONDS: u64 = 2_147_483_647; pub const TARGET_SALE_VALUE_KRW: u64 = 2_000_000_000; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -382,6 +385,8 @@ pub fn validate_threat(indicator: &ThreatIndicator) -> Result<(), &'static str> Ok(()) } +/// Validate DNSBL metadata, address-family prefix, loopback answer and cache +/// lifetime. Wardnet requires a positive TTL within RFC 2181's 31-bit range. pub fn validate_dnsbl(entry: &DnsblEntry) -> Result<(), &'static str> { if entry.reason.trim().is_empty() { return Err("DNSBL reason is required"); @@ -392,6 +397,9 @@ pub fn validate_dnsbl(entry: &DnsblEntry) -> Result<(), &'static str> { if entry.ttl_seconds == 0 { return Err("DNSBL ttl_seconds must be greater than 0"); } + if entry.ttl_seconds > DNSBL_MAX_TTL_SECONDS { + return Err("DNSBL ttl_seconds must not exceed 2147483647"); + } if let Some(prefix) = entry.prefix_len { let max = if entry.address.is_ipv4() { 32 } else { 128 }; if prefix > max { @@ -1396,9 +1404,34 @@ pub fn readiness_check(id: &str, passed: bool, evidence: &str) -> ReadinessCheck /// Export IPv4 DNSBL entries with loopback answers and lossless TXT metadata. /// Each TXT character string is at most 255 decoded UTF-8 bytes (RFC 1035 /// sections 3.3 and 3.3.14); longer metadata uses adjacent quoted strings. +/// Valid entries sharing an IPv4 owner use their shortest TTL for both RRsets; +/// persisted zero/out-of-range lifetimes are omitted rather than clamped. pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { let mut out = format!("$ORIGIN {}.\n$TTL 300\n", sanitize_zone_origin(origin)); + // Source-owned entries can share a published owner. RFC 2181 section 5.2 + // requires one TTL per RRset; use the shortest valid published lifetime, + // without changing stored source evidence or depending on input order. + let mut owner_ttls = std::collections::HashMap::::new(); + for entry in entries { + let IpAddr::V4(address) = entry.address else { + continue; + }; + let Ok(IpAddr::V4(code)) = IpAddr::from_str(&entry.code) else { + continue; + }; + if code.octets()[0] == 127 && (1..=DNSBL_MAX_TTL_SECONDS).contains(&entry.ttl_seconds) { + owner_ttls + .entry(address) + .and_modify(|ttl| *ttl = (*ttl).min(entry.ttl_seconds)) + .or_insert(entry.ttl_seconds); + } + } for entry in entries { + // Persisted input bypasses admission; never publish a lifetime that is + // zero or outside RFC 2181's 31-bit wire range. + if entry.ttl_seconds == 0 || entry.ttl_seconds > DNSBL_MAX_TTL_SECONDS { + continue; + } if let IpAddr::V4(address) = entry.address { // The response code is emitted as a bare, unquoted A-record token, so // it must be a valid IPv4 loopback literal (RFC 5782: DNSBL answers @@ -1414,9 +1447,17 @@ pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { _ => continue, }; let name = reverse_ipv4_for_dnsbl(address.octets()); - out.push_str(&format!("{} IN A {}\n", name, code)); + // Keep the existing 300-second representation byte-compatible, but + // publish other admitted cache lifetimes explicitly on both records. + let owner_ttl = owner_ttls[&address]; + let ttl = if owner_ttl == 300 { + String::new() + } else { + format!("{owner_ttl} ") + }; + out.push_str(&format!("{name} {ttl}IN A {code}\n")); out.push_str(&format!( - "{} IN TXT \"{}\"\n", + "{} {ttl}IN TXT \"{}\"\n", name, escape_txt(&format!("{} source={}", entry.reason, entry.source)) )); @@ -1488,6 +1529,76 @@ mod dnsbl_txt; mod tests { use super::*; + /// Exercise the validation matrix in the unit-test library instantiation, + /// including existing non-TTL admission invariants and the new wire bound. + #[test] + fn dnsbl_admission_validates_metadata_prefix_code_and_ttl() { + let base = AppData::seeded().dnsbl.remove(0); + assert_eq!(validate_dnsbl(&base), Ok(())); + let mut cases = Vec::new(); + let mut entry = base.clone(); + entry.reason = " ".into(); + cases.push((entry, "DNSBL reason is required")); + let mut entry = base.clone(); + entry.source = " ".into(); + cases.push((entry, "DNSBL source is required")); + let mut entry = base.clone(); + entry.ttl_seconds = 0; + cases.push((entry, "DNSBL ttl_seconds must be greater than 0")); + let mut entry = base.clone(); + entry.ttl_seconds = 2_147_483_648; + cases.push((entry, "DNSBL ttl_seconds must not exceed 2147483647")); + let mut entry = base.clone(); + entry.prefix_len = Some(33); + cases.push((entry, "DNSBL prefix_len exceeds the address family width")); + for (code, error) in [ + ("8.8.8.8", "DNSBL response code must be in 127.0.0.0/8"), + ( + "::1", + "DNSBL response code must be an IPv4 loopback address", + ), + ("invalid", "DNSBL response code must be an IP address"), + ] { + let mut entry = base.clone(); + entry.code = code.into(); + cases.push((entry, error)); + } + for (entry, error) in cases { + assert_eq!(validate_dnsbl(&entry), Err(error)); + } + let mut v6 = base; + v6.address = "2001:db8::1".parse().unwrap(); + v6.prefix_len = Some(128); + assert_eq!(validate_dnsbl(&v6), Ok(())); + v6.prefix_len = Some(129); + assert_eq!( + validate_dnsbl(&v6), + Err("DNSBL prefix_len exceeds the address family width") + ); + } + + /// Unit-library coverage control for duplicate-owner minimums, explicit + /// lifetimes and invalid persisted state. This is not another product RED. + #[test] + fn dnsbl_unit_publication_covers_shared_and_invalid_cache_lifetimes() { + let mut first = AppData::seeded().dnsbl.remove(0); + first.ttl_seconds = 600; + let mut second = first.clone(); + second.ttl_seconds = 60; + second.source = "second-source".into(); + let mut zero = first.clone(); + zero.ttl_seconds = 0; + let mut too_large = first.clone(); + too_large.ttl_seconds = u64::MAX; + let zone = export_dnsbl_zone("dnsbl.example", &[first, second, zero, too_large]); + assert_eq!(zone.lines().count(), 6); + assert!( + zone.lines() + .skip(2) + .all(|line| line.starts_with("10.113.0.203 60 IN ")) + ); + } + #[test] fn score_request_matches_client_ip_threat_indicators() { let threats = vec![ThreatIndicator { diff --git a/crates/waf-ids-core/tests/dnsbl_ttl.rs b/crates/waf-ids-core/tests/dnsbl_ttl.rs new file mode 100644 index 00000000..32850de2 --- /dev/null +++ b/crates/waf-ids-core/tests/dnsbl_ttl.rs @@ -0,0 +1,103 @@ +//! DNSBL cache-lifetime regressions at the zone publishing boundary. + +use waf_ids_core::{DnsblEntry, export_dnsbl_zone, validate_dnsbl}; + +/// Build an exact IPv4 entry with an independently selected cache lifetime. +fn entry(ttl_seconds: u64) -> DnsblEntry { + DnsblEntry { + address: "192.0.2.10".parse().unwrap(), + code: "127.0.0.2".to_string(), + reason: "scanner".to_string(), + source: "unit".to_string(), + ttl_seconds, + prefix_len: None, + } +} + +/// Resolve explicit/default TTLs from the records actually published by Wardnet. +fn record_ttls(zone: &str) -> Vec { + let default = zone + .lines() + .find_map(|line| line.strip_prefix("$TTL ")) + .unwrap() + .parse::() + .unwrap(); + zone.lines() + .filter(|line| !line.starts_with('$')) + .map(|line| { + let fields: Vec<_> = line.split_whitespace().collect(); + if fields[1] == "IN" { + default + } else { + assert_eq!(fields[2], "IN"); + fields[1].parse().unwrap() + } + }) + .collect() +} + +#[test] +fn published_a_and_txt_records_preserve_the_admitted_ttl() { + for ttl in [1, 60, 299, 300, 301, 86_400, 2_147_483_647] { + let entry = entry(ttl); + validate_dnsbl(&entry).unwrap(); + let zone = export_dnsbl_zone("dnsbl.example", &[entry]); + assert_eq!(record_ttls(&zone), [ttl, ttl], "zone: {zone}"); + } +} + +#[test] +fn ttl_outside_the_dns_wire_range_is_rejected_before_admission() { + for ttl in [2_147_483_648, u32::MAX as u64, u64::MAX] { + assert_eq!( + validate_dnsbl(&entry(ttl)), + Err("DNSBL ttl_seconds must not exceed 2147483647") + ); + } +} + +#[test] +fn invalid_persisted_ttl_cannot_make_the_published_zone_unloadable() { + for ttl in [0, 2_147_483_648, u64::MAX] { + let zone = export_dnsbl_zone("dnsbl.example", &[entry(ttl)]); + assert_eq!(zone, "$ORIGIN dnsbl.example.\n$TTL 300\n"); + } +} + +#[test] +fn a_shared_dns_owner_uses_the_shortest_valid_ttl_for_both_rrsets() { + let mut short = entry(60); + short.code = "127.0.0.3".to_string(); + short.source = "second-source".to_string(); + let mut invalid = entry(1); + invalid.code = "8.8.8.8".to_string(); + let mut other = entry(86_400); + other.address = "192.0.2.20".parse().unwrap(); + for entries in [ + vec![entry(600), short.clone(), invalid.clone(), other.clone()], + vec![other, invalid, short, entry(600)], + ] { + let zone = export_dnsbl_zone("dnsbl.example", &entries); + let shared: Vec<_> = zone + .lines() + .filter(|line| line.starts_with("10.2.0.192 ")) + .collect(); + assert_eq!(shared.len(), 4); + assert!( + shared + .iter() + .all(|line| line.starts_with("10.2.0.192 60 IN ")), + "{zone}" + ); + assert!(zone.contains("20.2.0.192 86400 IN A 127.0.0.2")); + assert!(!zone.contains("8.8.8.8")); + } +} + +#[test] +fn default_300_second_zone_remains_byte_identical() { + assert_eq!( + export_dnsbl_zone("dnsbl.example", &[entry(300)]), + "$ORIGIN dnsbl.example.\n$TTL 300\n10.2.0.192 IN A 127.0.0.2\n10.2.0.192 IN TXT \"scanner source=unit\"\n" + ); +} diff --git a/crates/waf-ids-core/tests/fuzz_invariants.rs b/crates/waf-ids-core/tests/fuzz_invariants.rs index f55f0a00..955bb4db 100644 --- a/crates/waf-ids-core/tests/fuzz_invariants.rs +++ b/crates/waf-ids-core/tests/fuzz_invariants.rs @@ -55,7 +55,7 @@ fn dnsbl_strategy() -> impl Strategy { dnsbl_code_strategy(), ".*", ".*", - any::(), + prop_oneof![any::(), 1u64..=2_147_483_647, Just(0), Just(300)], ) .prop_map(|(addr, code, reason, source, ttl_seconds)| DnsblEntry { address: IpAddr::V4(Ipv4Addr::from(addr)), @@ -100,8 +100,8 @@ proptest! { } } - // DNSBL classification and zone generation must never panic, and every TXT - // payload must be fully escaped (no unescaped double quote survives). + // DNSBL classification and zone generation must never panic; TXT payloads + // must retain valid escaped strings and legal adjacent-string delimiters. #[test] fn dnsbl_zone_generation_escapes_and_never_panics( origin in ".*", @@ -113,23 +113,17 @@ proptest! { let zone = export_dnsbl_zone(&origin, &entries); prop_assert!(zone.starts_with("$ORIGIN ")); - // Every published A-record response code is an IPv4 loopback literal - // (RFC 5782 / the "response code in 127.0.0.0/8" invariant); non-127/8 - // or IPv6 codes must be dropped, never emitted. Parse by record structure - // (` IN A ` = four whitespace fields) so a TXT line whose - // escaped reason/source payload contains the substring " IN A " is never - // misread as an A-record. + // Parse optional explicit TTL before class/type so TXT content cannot + // masquerade as an A record. Both record forms keep the 127/8 check. for line in zone.lines() { - let mut fields = line.split_whitespace(); - let (Some(_name), Some("IN"), Some("A"), Some(code), None) = ( - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - ) else { - continue; + let fields: Vec<_> = line.split_whitespace().collect(); + let record = match fields.as_slice() { + [_name, "IN", "A", code] => Some((300, *code)), + [_name, ttl, "IN", "A", code] => Some((ttl.parse::().unwrap(), *code)), + _ => None, }; + let Some((ttl, code)) = record else { continue }; + prop_assert!((1..=2_147_483_647).contains(&ttl)); match code.parse::() { Ok(IpAddr::V4(v4)) => { prop_assert_eq!(v4.octets()[0], 127, "non-loopback A code: {}", code) @@ -141,7 +135,8 @@ proptest! { dnsbl_txt::assert_zone_txt_valid(&zone); let txt_lines: Vec<_> = zone.lines().filter_map(|l| l.split_once(" IN TXT ")).collect(); let expected: Vec<_> = entries.iter().filter(|e| { - matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127) + (1..=2_147_483_647).contains(&e.ttl_seconds) + && matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127) }).collect(); prop_assert_eq!(txt_lines.len(), expected.len()); for ((_, text), entry) in txt_lines.iter().zip(expected) { diff --git a/docs/doctoring/dnsbl-cache-lifetimes.md b/docs/doctoring/dnsbl-cache-lifetimes.md new file mode 100644 index 00000000..34f42e5b --- /dev/null +++ b/docs/doctoring/dnsbl-cache-lifetimes.md @@ -0,0 +1,48 @@ +# DNSBL cache lifetimes at the publication boundary + +## Reproduced product defect + +The management API stored `DnsblEntry.ttl_seconds`, but the DNSBL exporter +omitted every record TTL and supplied `$TTL 300`. An admitted 1-second entry +therefore published A and TXT records with 300-second effective cache lifetimes. +The observed regression failed with `[300, 300]` instead of `[1, 1]` before the +production repair. This is DNS publication, not threat-feed expiry enforcement. + +## Contract and minimal repair + +DNS TTLs are unsigned values from zero through 2147483647. The high wire bit +must be zero. Wardnet already rejects zero at DNSBL admission; this repair adds +the upper-bound rejection without changing the separate threat-indicator or +feed-status lifetime contracts.[1] + +An isolated DNSBL owner's A and TXT records preserve the admitted lifetime. +Entries sharing an IPv4 owner form A and TXT RRsets. All records in each RRset +must use the same TTL; Wardnet selects the shortest valid published lifetime, +independent of input order, without changing source-owned stored evidence.[1] +Invalid loopback codes, IPv6 entries and invalid TTLs do not contribute to this +minimum. This publication projection does not adopt another feed owner's work. + +Persisted state is an untrusted boundary that bypasses admission. The exporter +omits entries with zero or out-of-range TTLs rather than silently clamping them. +For an effective 300-second owner, existing zone bytes remain unchanged; other +lifetimes are explicit on both record types. Metadata escaping/chunking and the +existing response-code safeguards remain intact. + +## Verification and acceptance limits + +The new core regression covers 1/60/299/300/301/86400/2147483647 seconds, +upper-range admission rejection, invalid persisted TTL omission, byte-identical +300-second output and shared-owner minimum TTL under both input orders. +The HTTP regression exercises authenticated admission, readback, upsert and +zone export, including rejected mutations preserving the previous valid entry. +Stable property tests retain arbitrary u64 TTLs and add valid-range and default +controls; the synchronized fuzz oracle recognizes explicit TTL syntax. + +This repair does not establish authoritative DNS deployment, SOA/NS provisioning, +DNS name limits, aggregate TXT RDLENGTH bounds, CIDR or IPv6 publishing, +threat/feed expiration, complete product coverage, performance or business +acceptance. A local review is not a counted GitHub approval or protected merge. + +## Sources + +[1] https://www.rfc-editor.org/rfc/rfc2181.html โ€” Elz & Bush (1997). Clarifications to the DNS Specification (RFC 2181), sections 5.2 and 8 diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 42c87838..93e4c229 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -19,7 +19,7 @@ entry points for arbitrary input. | `fuzz_score_request` | `waf_ids_core::score_request` | no panic on arbitrary path/query/body/IP; `reason` never empty; scoring deterministic | | `fuzz_appdata_json` | `serde_json::from_str::` (state file) | no panic; parsed values round-trip through serde | | `fuzz_parse_admin_tokens` | `waf_ids_ai_soc::parse_admin_tokens` | no panic; no empty token key; no empty principal actor value | -| `fuzz_dnsbl_zone` | `waf_ids_core::export_dnsbl_zone` / `validate_dnsbl` | no panic; quoted TXT strings decode losslessly, remain injection-safe and contain at most 255 decoded bytes each (RFC 1035 ยง3.3/ยง3.3.14); every published A-record response code is an IPv4 loopback literal (127.0.0.0/8) | +| `fuzz_dnsbl_zone` | `waf_ids_core::export_dnsbl_zone` / `validate_dnsbl` | no panic; quoted TXT strings decode losslessly, remain injection-safe and contain at most 255 decoded bytes each (RFC 1035 ยง3.3/ยง3.3.14); every published A-record response code is an IPv4 loopback literal (127.0.0.0/8); published TTLs are in 1..=2147483647, while invalid persisted TTLs are omitted | ## Layout diff --git a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs index e310efa5..94084d32 100644 --- a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs +++ b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs @@ -85,24 +85,17 @@ fuzz_target!(|input: Input| { "zone must start with $ORIGIN directive" ); - // Every published A-record response code is an IPv4 loopback literal - // (RFC 5782 / the "response code in 127.0.0.0/8" invariant). Codes outside - // 127/8, IPv6 literals, and unparseable strings must all be dropped. Parse - // by record structure from the start of the line: an A record is exactly - // ` IN A ` (four whitespace fields, no spaces in name or code). - // Matching on the substring " IN A " instead would misread a TXT line whose - // escaped reason/source payload merely contains that substring as an A-record. + // Parse optional explicit TTL before class/type; TXT content must never + // masquerade as an A record, and every answer retains its 127/8 check. for line in zone.lines() { - let mut fields = line.split_whitespace(); - let (Some(_name), Some("IN"), Some("A"), Some(code), None) = ( - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - ) else { - continue; + let fields: Vec<_> = line.split_whitespace().collect(); + let record = match fields.as_slice() { + [_name, "IN", "A", code] => Some((300, *code)), + [_name, ttl, "IN", "A", code] => Some((ttl.parse::().unwrap(), *code)), + _ => None, }; + let Some((ttl, code)) = record else { continue }; + assert!((1..=2_147_483_647).contains(&ttl)); match code.parse::() { Ok(IpAddr::V4(v4)) => assert_eq!( v4.octets()[0], @@ -122,7 +115,10 @@ fuzz_target!(|input: Input| { .collect(); let expected: Vec<_> = entries .iter() - .filter(|e| matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127)) + .filter(|e| { + (1..=2_147_483_647).contains(&e.ttl_seconds) + && matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127) + }) .collect(); assert_eq!(txt_lines.len(), expected.len()); for ((_, text), entry) in txt_lines.iter().zip(expected) { diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 0cc18e0f..d2a230a2 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -209,7 +209,8 @@ fi zone="$(curl -fsS "$BASE_URL/dnsbl/zone")" grep -q '^\$ORIGIN dnsbl.test\.$' <<<"$zone" grep -q '^10.113.0.203 IN A 127.0.0.2$' <<<"$zone" -grep -q '^23.100.51.198 IN A 127.0.0.4$' <<<"$zone" +grep -q '^23.100.51.198 600 IN A 127.0.0.4$' <<<"$zone" +grep -q '^23.100.51.198 600 IN TXT "feed scanner source=misp-seoul"$' <<<"$zone" kill "$SERVER_PID" wait "$SERVER_PID" 2>/dev/null || true diff --git a/tests/dnsbl_ttl_export.rs b/tests/dnsbl_ttl_export.rs new file mode 100644 index 00000000..d37503b2 --- /dev/null +++ b/tests/dnsbl_ttl_export.rs @@ -0,0 +1,100 @@ +//! DNSBL API admission, readback and exported cache-lifetime contract. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +/// Exercise real authenticated routes with a disposable in-memory aggregate. +#[tokio::test] +async fn dnsbl_ttl_is_preserved_from_admission_to_zone_export() { + let app = build_app(AppState::seeded(Some("ttl-fixture-token".to_string()))); + for (ttl, expected_status) in [ + (60u64, StatusCode::CREATED), + (2_147_483_647, StatusCode::CREATED), + (2_147_483_648, StatusCode::BAD_REQUEST), + (u64::MAX, StatusCode::BAD_REQUEST), + (0, StatusCode::BAD_REQUEST), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/dnsbl") + .header("content-type", "application/json") + .header("x-admin-token", "ttl-fixture-token") + .body(Body::from( + serde_json::json!({ + "address": "192.0.2.10", "code": "127.0.0.2", + "reason": "scanner", "source": "unit", "ttl_seconds": ttl + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), expected_status); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/api/dnsbl") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let entries: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let saved = entries + .as_array() + .unwrap() + .iter() + .find(|entry| entry["address"] == "192.0.2.10") + .unwrap(); + let expected_ttl = if expected_status == StatusCode::CREATED { + ttl + } else { + 2_147_483_647 + }; + assert_eq!(saved["ttl_seconds"], expected_ttl); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/dnsbl/zone") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let zone = String::from_utf8( + to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() + .to_vec(), + ) + .unwrap(); + assert!( + zone.contains(&format!("10.2.0.192 {expected_ttl} IN A 127.0.0.2\n")), + "{zone}" + ); + assert!( + zone.contains(&format!( + "10.2.0.192 {expected_ttl} IN TXT \"scanner source=unit\"\n" + )), + "{zone}" + ); + assert_eq!( + zone.lines().count(), + 6, + "rejected writes must not add records" + ); + } +} From adf4fda748904c2af8cd9623ddaa9291caed09ef Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 01:04:20 +0900 Subject: [PATCH 03/22] test(dnsbl): enforce independent publication oracle sensitivity --- .../waf-ids-core/tests/dnsbl_zone_oracle.rs | 95 +++++++++++++++++++ crates/waf-ids-core/tests/fuzz_invariants.rs | 39 +++++++- .../waf-ids-core/tests/support/dnsbl_zone.rs | 88 +++++++++++++++++ docs/fuzzing.md | 11 +++ fuzz/fuzz_targets/fuzz_dnsbl_zone.rs | 29 +++++- fuzz/support/dnsbl_zone.rs | 6 ++ 6 files changed, 263 insertions(+), 5 deletions(-) create mode 100644 crates/waf-ids-core/tests/dnsbl_zone_oracle.rs create mode 100644 crates/waf-ids-core/tests/support/dnsbl_zone.rs create mode 100644 fuzz/support/dnsbl_zone.rs diff --git a/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs b/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs new file mode 100644 index 00000000..ce761b7f --- /dev/null +++ b/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs @@ -0,0 +1,95 @@ +//! Independent publication-oracle sensitivity, not synthetic product defects. + +#[path = "support/dnsbl_zone.rs"] +mod dnsbl_zone; + +use waf_ids_core::DnsblEntry; + +/// Build two source records sharing an IPv4 DNS owner with different TTLs. +fn entries() -> Vec { + [600, 60] + .into_iter() + .map(|ttl_seconds| DnsblEntry { + address: "192.0.2.10".parse().unwrap(), + code: "127.0.0.2".into(), + reason: "scanner".into(), + source: "unit".into(), + ttl_seconds, + prefix_len: None, + }) + .collect() +} + +/// Render explicit independent fixture records rather than calling the exporter. +fn zone(ttls: [u64; 2]) -> String { + let mut zone = "$ORIGIN dnsbl.example.\n$TTL 300\n".to_string(); + for ttl in ttls { + zone.push_str(&format!( + "10.2.0.192 {ttl} IN A 127.0.0.2\n10.2.0.192 {ttl} IN TXT \"scanner source=unit\"\n" + )); + } + zone +} + +/// Accept shared-owner minimums and implicit default TTLs as positive controls. +#[test] +fn oracle_accepts_valid_shared_owner_minimum_and_default_ttl() { + dnsbl_zone::assert_zone_matches_entries(&zone([60, 60]), &entries()); + let mut entries = entries(); + for entry in &mut entries { + entry.ttl_seconds = 300; + } + dnsbl_zone::assert_zone_matches_entries( + &zone([300, 300]).replace(" 300 IN ", " IN "), + &entries, + ); +} + +/// Reject malformed cache lifetimes while requiring valid controls to pass. +#[test] +fn oracle_rejects_wrong_or_inconsistent_cache_lifetimes() { + for bad in [zone([600, 60]), zone([300, 300]), zone([0, 0])] { + assert!( + std::panic::catch_unwind(|| { + dnsbl_zone::assert_zone_matches_entries(&bad, &entries()); + }) + .is_err(), + "oracle admitted incorrect TTLs: {bad}" + ); + } +} + +/// Reject missing records even when the surviving record grammar is valid. +#[test] +fn oracle_rejects_missing_answer_or_text_records() { + for omitted in [" IN A ", " IN TXT "] { + let bad = zone([60, 60]) + .lines() + .filter(|line| !line.contains(omitted)) + .collect::>() + .join("\n"); + assert!( + std::panic::catch_unwind(|| { + dnsbl_zone::assert_zone_matches_entries(&bad, &entries()); + }) + .is_err() + ); + } +} + +/// Reject otherwise-legal answer owner, response-code and metadata drift. +#[test] +fn oracle_rejects_answer_identity_and_metadata_drift() { + for bad in [ + zone([60, 60]).replace("127.0.0.2", "127.0.0.3"), + zone([60, 60]).replace("10.2.0.192", "11.2.0.192"), + zone([60, 60]).replace("scanner source=unit", "other source=unit"), + ] { + assert!( + std::panic::catch_unwind(|| { + dnsbl_zone::assert_zone_matches_entries(&bad, &entries()); + }) + .is_err() + ); + } +} diff --git a/crates/waf-ids-core/tests/fuzz_invariants.rs b/crates/waf-ids-core/tests/fuzz_invariants.rs index 955bb4db..60a79f88 100644 --- a/crates/waf-ids-core/tests/fuzz_invariants.rs +++ b/crates/waf-ids-core/tests/fuzz_invariants.rs @@ -6,8 +6,10 @@ //! without a nightly toolchain. The fuzz targets explore far deeper; these keep //! a fast, always-green signal. -#[path = "support/dnsbl_txt.rs"] -mod dnsbl_txt; +#[path = "support/dnsbl_zone.rs"] +mod dnsbl_zone; + +use dnsbl_zone::dnsbl_txt; use proptest::prelude::*; use std::net::{IpAddr, Ipv4Addr}; @@ -100,6 +102,38 @@ proptest! { } } + // Unlike the broad mixed-input property below, force 2..8 publishable + // records to share one owner, with independently varied positive TTLs and + // metadata. The test-only oracle independently checks minimum-per-owner TTL, + // A/TXT parity, record identity/order, and lossless reason/source bytes. + #[test] + fn dnsbl_zone_preserves_shared_owner_source_metadata_and_minimum_ttl( + address in any::(), + records in proptest::collection::vec(( + 1u64..=2_147_483_647, + ".{0,600}", + ".{0,120}", + ), 2..8), + ) { + let address = IpAddr::V4(Ipv4Addr::from(address)); + let entries: Vec<_> = records + .into_iter() + .enumerate() + .map(|(index, (ttl_seconds, reason, source))| DnsblEntry { + address, + code: format!("127.0.0.{}", index + 1), + reason, + source, + ttl_seconds, + prefix_len: None, + }) + .collect(); + let original = entries.clone(); + let zone = export_dnsbl_zone("dnsbl.example", &entries); + dnsbl_zone::assert_zone_matches_entries(&zone, &entries); + prop_assert_eq!(entries, original, "export must not mutate source-owned entries"); + } + // DNSBL classification and zone generation must never panic; TXT payloads // must retain valid escaped strings and legal adjacent-string delimiters. #[test] @@ -112,6 +146,7 @@ proptest! { } let zone = export_dnsbl_zone(&origin, &entries); prop_assert!(zone.starts_with("$ORIGIN ")); + dnsbl_zone::assert_zone_matches_entries(&zone, &entries); // Parse optional explicit TTL before class/type so TXT content cannot // masquerade as an A record. Both record forms keep the 127/8 check. diff --git a/crates/waf-ids-core/tests/support/dnsbl_zone.rs b/crates/waf-ids-core/tests/support/dnsbl_zone.rs new file mode 100644 index 00000000..412a52a7 --- /dev/null +++ b/crates/waf-ids-core/tests/support/dnsbl_zone.rs @@ -0,0 +1,88 @@ +//! Test-only independent DNSBL publication oracle shared with the fuzz harness. + +#[path = "dnsbl_txt.rs"] +pub mod dnsbl_txt; + +use std::net::IpAddr; +use waf_ids_core::DnsblEntry; + +/// Check the exported records against input evidence without calling production +/// validation, address reversal or lifetime projection helpers. Expected TTLs +/// use an independent linear scan per owner rather than the exporter's map. +pub fn assert_zone_matches_entries(zone: &str, entries: &[DnsblEntry]) { + dnsbl_txt::assert_zone_txt_valid(zone); + let expected: Vec<_> = entries + .iter() + .filter(|entry| { + entry.address.is_ipv4() + && (1..=2_147_483_647).contains(&entry.ttl_seconds) + && matches!(entry.code.parse::(), Ok(IpAddr::V4(code)) if code.octets()[0] == 127) + }) + .collect(); + let mut lines = zone.lines(); + assert!(lines.next().unwrap().starts_with("$ORIGIN ")); + assert_eq!(lines.next(), Some("$TTL 300")); + let mut answers = Vec::new(); + let mut texts = Vec::new(); + for line in lines { + let (owner, record) = line + .split_once(" IN ") + .expect("record must contain IN class"); + let fields: Vec<_> = owner.split_whitespace().collect(); + let (name, ttl) = match fields.as_slice() { + [name] => (*name, 300), + [name, ttl] => (*name, ttl.parse::().expect("invalid explicit TTL")), + _ => panic!("invalid owner/TTL fields: {owner}"), + }; + assert!((1..=2_147_483_647).contains(&ttl)); + if let Some(code) = record.strip_prefix("A ") { + let IpAddr::V4(code) = code.parse::().expect("invalid A answer") else { + panic!("IPv6 answer in A record"); + }; + assert_eq!(code.octets()[0], 127); + answers.push((name, ttl, code)); + } else if let Some(text) = record.strip_prefix("TXT ") { + texts.push((name, ttl, dnsbl_txt::decode_txt_rdata(text).concat())); + } else { + panic!("unexpected DNSBL record: {record}"); + } + } + assert_eq!( + answers.len(), + expected.len(), + "A count must match input evidence" + ); + assert_eq!( + texts.len(), + expected.len(), + "TXT count must match input evidence" + ); + for ((answer, text), entry) in answers.iter().zip(&texts).zip(&expected) { + let IpAddr::V4(address) = entry.address else { + unreachable!() + }; + let [a, b, c, d] = address.octets(); + let name = format!("{d}.{c}.{b}.{a}"); + let ttl = expected + .iter() + .filter(|other| other.address == entry.address) + .map(|other| other.ttl_seconds) + .min() + .unwrap(); + assert_eq!(answer.0, name, "A owner/order mismatch"); + assert_eq!(text.0, name, "TXT owner/order mismatch"); + assert_eq!( + answer.1, ttl, + "A TTL must match owner's shortest input lifetime" + ); + assert_eq!( + text.1, ttl, + "TXT TTL must match owner's shortest input lifetime" + ); + assert_eq!(IpAddr::V4(answer.2), entry.code.parse::().unwrap()); + assert_eq!( + text.2, + format!("{} source={}", entry.reason, entry.source).as_bytes() + ); + } +} diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 93e4c229..61cda02e 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -34,6 +34,17 @@ The property-test mirror lives in `crates/waf-ids-core/tests/fuzz_invariants.rs` and `tests/fuzz_invariants.rs` (proptest); it enforces the same invariants on stable as part of `cargo test --workspace`. +DNSBL publication additionally has a package-local independent oracle in +`crates/waf-ids-core/tests/support/dnsbl_zone.rs`. It checks input-derived A/TXT +counts, owner and source order, lossless metadata and shortest valid TTL per +IPv4 owner without calling the production projection or its limit constant. +The stable shared-owner property always generates at least two publishable +records with valid TTLs; metadata may still contain arbitrary or empty text. +the fuzz harness retains its arbitrary-input pass and adds a bounded shared-owner +positive projection. `dnsbl_zone_oracle.rs` pairs legal controls with malformed +TTL, missing-record and identity/metadata negatives. These are test-sensitivity +checks, not new production defects or proof that a fuzz campaign ran. + ## Running locally Coverage-guided fuzzing needs a nightly toolchain: diff --git a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs index 94084d32..ac3961d1 100644 --- a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs +++ b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs @@ -11,13 +11,15 @@ //! at most 255 decoded bytes each; concatenation preserves reason/source bytes. //! Quotes, backslashes and control characters cannot break out into zone lines. -#[path = "../support/dnsbl_txt.rs"] -mod dnsbl_txt; +#[path = "../support/dnsbl_zone.rs"] +mod dnsbl_zone; + +use dnsbl_zone::dnsbl_txt; use arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use waf_ids_core::{DnsblEntry, export_dnsbl_zone, validate_dnsbl}; +use waf_ids_core::{export_dnsbl_zone, validate_dnsbl, DnsblEntry}; /// A response code drawn from the raw fuzz bytes: arbitrary strings plus real IP /// literals (loopback, non-loopback IPv4, IPv6) so the zone A-record invariant @@ -79,6 +81,27 @@ fuzz_target!(|input: Input| { // Zone generation must never panic on arbitrary strings. let zone = export_dnsbl_zone(&input.origin, &entries); + dnsbl_zone::assert_zone_matches_entries(&zone, &entries); + + // Retain the arbitrary-input pass above. Add a bounded positive projection + // with a shared owner, loopback codes and valid TTLs so almost-all-invalid + // arbitrary u64 lifetimes cannot make the evidence checks vacuous. + if let Some(first) = entries.first() { + let shared: Vec<_> = entries + .iter() + .take(8) + .enumerate() + .map(|(index, entry)| DnsblEntry { + address: first.address, + code: format!("127.0.0.{}", index + 1), + ttl_seconds: 1 + entry.ttl_seconds % 2_147_483_647, + ..entry.clone() + }) + .collect(); + let shared_zone = export_dnsbl_zone(&input.origin, &shared); + dnsbl_zone::assert_zone_matches_entries(&shared_zone, &shared); + } + // The zone always carries its header directive. assert!( zone.starts_with("$ORIGIN "), diff --git a/fuzz/support/dnsbl_zone.rs b/fuzz/support/dnsbl_zone.rs new file mode 100644 index 00000000..13f98137 --- /dev/null +++ b/fuzz/support/dnsbl_zone.rs @@ -0,0 +1,6 @@ +//! Fuzz-only access to the package-local independent publication oracle. + +#[path = "../../crates/waf-ids-core/tests/support/dnsbl_zone.rs"] +mod oracle; + +pub use oracle::{assert_zone_matches_entries, dnsbl_txt}; From e1e45997afc69ac22b4db2ed5d42df9aa1639383 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 04:28:38 +0900 Subject: [PATCH 04/22] fix(dnsbl): bound origin labels and full owner wire length --- crates/waf-ids-core/src/lib.rs | 15 ++++- crates/waf-ids-core/tests/dnsbl_origin.rs | 58 ++++++++++++++++++ .../waf-ids-core/tests/dnsbl_zone_oracle.rs | 37 ++++++++++++ .../waf-ids-core/tests/support/dnsbl_zone.rs | 16 ++++- docs/doctoring/dnsbl-origin-wire-limits.md | 54 +++++++++++++++++ docs/fuzzing.md | 8 ++- tests/dnsbl_origin_export.rs | 59 +++++++++++++++++++ 7 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 crates/waf-ids-core/tests/dnsbl_origin.rs create mode 100644 docs/doctoring/dnsbl-origin-wire-limits.md create mode 100644 tests/dnsbl_origin_export.rs diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index 8198395b..1bfb7464 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -1470,8 +1470,9 @@ pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { /// of the generated zone file. A legitimate origin is a domain name, so only /// letters, digits, `-`, `_`, and `.` are kept; every other byte (newline, /// quote, space, control char) is dropped. Leading/trailing dots are trimmed -/// because the caller re-appends the root dot. Empty input falls back to the -/// RFC 6761 reserved `.invalid` TLD, which is guaranteed non-resolvable. +/// because the caller re-appends the root dot. Empty labels, oversized labels +/// or an origin that cannot fit every reversed IPv4 owner fall back to the +/// RFC 6761 reserved `.invalid` TLD. Existing valid origin spelling is retained. fn sanitize_zone_origin(origin: &str) -> String { let filtered: String = origin .trim() @@ -1479,7 +1480,15 @@ fn sanitize_zone_origin(origin: &str) -> String { .filter(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) .collect(); let trimmed = filtered.trim_matches('.'); - if trimmed.is_empty() { + // A textual ASCII origin uses len + 2 wire octets (label lengths and + // root). Reserve 16 more for four maximum-length reversed IPv4 labels. + // RFC 1035 sections 2.3.4/3.1 cap the full owner at 255 wire octets. + if trimmed.is_empty() + || trimmed.len() > 237 + || trimmed + .split('.') + .any(|label| label.is_empty() || label.len() > 63) + { "dnsbl.invalid".to_string() } else { trimmed.to_string() diff --git a/crates/waf-ids-core/tests/dnsbl_origin.rs b/crates/waf-ids-core/tests/dnsbl_origin.rs new file mode 100644 index 00000000..979df7f8 --- /dev/null +++ b/crates/waf-ids-core/tests/dnsbl_origin.rs @@ -0,0 +1,58 @@ +//! Publication boundary regressions for DNS label and full-owner wire limits. + +use waf_ids_core::{DnsblEntry, export_dnsbl_zone}; + +fn entry() -> DnsblEntry { + DnsblEntry { + address: "255.255.255.255".parse().unwrap(), + code: "127.0.0.2".into(), + reason: "scanner".into(), + source: "unit".into(), + ttl_seconds: 300, + prefix_len: None, + } +} + +#[test] +fn origin_reserves_wire_space_for_the_longest_ipv4_owner() { + // Four reversed IPv4 labels each need one length octet plus three digits. + // These origins consume 239 and 240 wire bytes including the root octet. + let valid = [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(45), + ] + .join("."); + let invalid = [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(46), + ] + .join("."); + assert_eq!(valid.len() + 2 + 16, 255); + assert_eq!(invalid.len() + 2 + 16, 256); + assert!(export_dnsbl_zone(&valid, &[entry()]).starts_with(&format!("$ORIGIN {valid}.\n"))); + let zone = export_dnsbl_zone(&invalid, &[entry()]); + assert!(zone.starts_with("$ORIGIN dnsbl.invalid.\n")); + assert!(zone.contains("255.255.255.255 IN A 127.0.0.2\n")); +} + +#[test] +fn interior_empty_origin_label_uses_existing_nonresolving_fallback() { + let zone = export_dnsbl_zone("dnsbl..example", &[entry()]); + assert!(zone.starts_with("$ORIGIN dnsbl.invalid.\n")); + assert!(zone.contains("255.255.255.255 IN A 127.0.0.2\n")); +} + +#[test] +fn oversized_origin_label_uses_existing_nonresolving_fallback() { + let valid = format!("{}.example", "a".repeat(63)); + assert!(export_dnsbl_zone(&valid, &[entry()]).starts_with(&format!("$ORIGIN {valid}.\n"))); + let invalid = format!("{}.example", "a".repeat(64)); + let zone = export_dnsbl_zone(&invalid, &[entry()]); + assert!(zone.starts_with("$ORIGIN dnsbl.invalid.\n")); + assert!(zone.contains("255.255.255.255 IN A 127.0.0.2\n")); + assert!(zone.contains("255.255.255.255 IN TXT \"scanner source=unit\"\n")); +} diff --git a/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs b/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs index ce761b7f..8ec3f285 100644 --- a/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs +++ b/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs @@ -45,6 +45,43 @@ fn oracle_accepts_valid_shared_owner_minimum_and_default_ttl() { ); } +/// Reject unparseable origins with otherwise-valid records and metadata. +#[test] +fn oracle_rejects_unparseable_origin_names() { + // A 237-character origin plus the longest IPv4 owner fits exactly. + let valid_origin = [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(45), + ] + .join("."); + dnsbl_zone::assert_zone_matches_entries( + &zone([60, 60]).replace("dnsbl.example.", &format!("{valid_origin}.")), + &entries(), + ); + for origin in [ + format!("{}.example", "a".repeat(64)), + "dnsbl..example".into(), + [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(46), + ] + .join("."), + ] { + let bad = zone([60, 60]).replace("dnsbl.example.", &format!("{origin}.")); + assert!( + std::panic::catch_unwind(|| { + dnsbl_zone::assert_zone_matches_entries(&bad, &entries()); + }) + .is_err(), + "oracle admitted invalid origin: {origin}" + ); + } +} + /// Reject malformed cache lifetimes while requiring valid controls to pass. #[test] fn oracle_rejects_wrong_or_inconsistent_cache_lifetimes() { diff --git a/crates/waf-ids-core/tests/support/dnsbl_zone.rs b/crates/waf-ids-core/tests/support/dnsbl_zone.rs index 412a52a7..990b5550 100644 --- a/crates/waf-ids-core/tests/support/dnsbl_zone.rs +++ b/crates/waf-ids-core/tests/support/dnsbl_zone.rs @@ -20,7 +20,21 @@ pub fn assert_zone_matches_entries(zone: &str, entries: &[DnsblEntry]) { }) .collect(); let mut lines = zone.lines(); - assert!(lines.next().unwrap().starts_with("$ORIGIN ")); + let origin = lines.next().unwrap().strip_prefix("$ORIGIN ").unwrap(); + let labels: Vec<_> = origin.strip_suffix('.').unwrap().split('.').collect(); + assert!(labels.iter().all(|label| { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + })); + // Calculate encoded label lengths independently of the sanitizer's text cap. + let origin_wire_bytes = 1 + labels.iter().map(|label| 1 + label.len()).sum::(); + assert!( + origin_wire_bytes + 4 * (1 + 3) <= 255, + "origin cannot fit every IPv4 owner" + ); assert_eq!(lines.next(), Some("$TTL 300")); let mut answers = Vec::new(); let mut texts = Vec::new(); diff --git a/docs/doctoring/dnsbl-origin-wire-limits.md b/docs/doctoring/dnsbl-origin-wire-limits.md new file mode 100644 index 00000000..944310b3 --- /dev/null +++ b/docs/doctoring/dnsbl-origin-wire-limits.md @@ -0,0 +1,54 @@ +# DNSBL origin and owner wire limits + +## Publication defect + +The origin sanitizer removed injection characters but accepted interior empty +labels, labels longer than 63 bytes, and origins that left no wire space for a +reversed IPv4 owner. The actual Rust exporter fed to dnspython 2.8.0 reproduced +`LabelTooLong`, `EmptyLabel`, and `NameTooLong`. A positive control with a +255-byte fully qualified owner parsed; the otherwise identical 256-byte owner +failed. These are publication defects, not authoritative-server deployment +observations. + +## Bounded repair and compatibility + +Keep the existing ASCII filtering, surrounding-dot trimming and ordinary +origin spelling. After filtering, use the existing `dnsbl.invalid` fallback +when any label is empty or longer than 63 bytes, or when the origin exceeds +237 textual bytes. The fallback changes the published origin, not stored +entries, their metadata, response codes, order or cache lifetimes. It is not +an operator configuration-admission error or a new public API. + +For a nonempty dotted ASCII name without its final root dot, encoded length +is textual length plus two: separators become length octets, the first label +adds one length octet, and the root adds one zero octet. Four reversed IPv4 +labels require at most sixteen more octets. Therefore the bound is +`237 + 2 + 16 = 255`. Reserving the maximum preserves a stable origin for all +IPv4 entries rather than changing it with the current address set. + +The shared package-local test oracle independently sums label lengths and +length octets rather than using the sanitizer's textual threshold. Fixed +63/64-byte label and 255/256-byte owner controls, arbitrary-input properties, +the synchronized fuzz oracle, and the actual configured HTTP export route +exercise this boundary. Scratch compilation is not a fuzz campaign. + +## Scope and remaining gaps + +This repair is in DNSBL publication, not the separately owned runtime config +registry or feed-refresh work. It does not establish SOA/NS completeness, +authoritative loading, aggregate TXT RDLENGTH, IPv6/CIDR publication, UI locale +rendering, production rollout or full-workspace 100% coverage. Local tests and +source review do not replace current-head hosted security gates or counted +GitHub approval. + +## Source + +Mockapetris, P. (November 1987). *Domain names โ€” implementation and +specification*. RFC 1035, sections 2.3.4 and 3.1. +https://www.rfc-editor.org/rfc/rfc1035.txt + +The RFC defines 63-octet labels, a 255-octet encoded domain name, label-length +fields and the terminating root label. The sixteen-octet IPv4 reserve and +237-character publication threshold above are Wardnet's derived boundary, +not verbatim RFC limits. The official text permits unlimited distribution; +no third-party paper or implementation is copied by this repair. diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 61cda02e..a4236abc 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -42,8 +42,12 @@ The stable shared-owner property always generates at least two publishable records with valid TTLs; metadata may still contain arbitrary or empty text. the fuzz harness retains its arbitrary-input pass and adds a bounded shared-owner positive projection. `dnsbl_zone_oracle.rs` pairs legal controls with malformed -TTL, missing-record and identity/metadata negatives. These are test-sensitivity -checks, not new production defects or proof that a fuzz campaign ran. +TTL, missing-record and identity/metadata negatives. The same oracle checks +ASCII origin labels of 1..=63 bytes and independently computes the encoded +origin length, including label-length and root octets, with room for all four +reversed IPv4 labels. Invalid origins use the existing `dnsbl.invalid` fallback; +ordinary origin spelling and record evidence remain unchanged. Oracle negatives +are test-sensitivity checks, not proof that a fuzz campaign ran. ## Running locally diff --git a/tests/dnsbl_origin_export.rs b/tests/dnsbl_origin_export.rs new file mode 100644 index 00000000..0a6cbbf6 --- /dev/null +++ b/tests/dnsbl_origin_export.rs @@ -0,0 +1,59 @@ +//! Config-to-HTTP DNSBL publication boundary without an operational server. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppConfig, AppState, build_app}; + +/// Invalid operator origins must not produce an unloadable publication response. +#[tokio::test] +async fn configured_dnsbl_origin_is_checked_at_the_http_publication_boundary() { + let cases = [ + ("dnsbl.example.".to_string(), "dnsbl.example".to_string()), + ("dnsbl..example".to_string(), "dnsbl.invalid".to_string()), + ( + format!("{}.example", "a".repeat(64)), + "dnsbl.invalid".to_string(), + ), + ( + [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(46), + ] + .join("."), + "dnsbl.invalid".to_string(), + ), + ]; + for (origin, expected) in cases { + let mut config = AppConfig::memory(None); + config.dnsbl_origin = origin; + let state = AppState::load(config).await.unwrap(); + let app = build_app(state); + let response = app + .oneshot( + Request::builder() + .uri("/dnsbl/zone") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()["content-type"], + "text/plain; charset=utf-8" + ); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let zone = std::str::from_utf8(&body).unwrap(); + assert!( + zone.starts_with(&format!("$ORIGIN {expected}.\n")), + "{zone}" + ); + assert!(zone.contains("IN A 127.0.0.2")); + assert!(zone.contains("IN TXT ")); + } +} From c0b83f0b05169c4e38fe4ef0aed11e672c72268a Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 05:17:43 +0900 Subject: [PATCH 05/22] fix(dnsbl): bound aggregate TXT RDATA wire length --- crates/waf-ids-core/src/lib.rs | 65 ++++++++- crates/waf-ids-core/tests/dnsbl_rdlength.rs | 134 ++++++++++++++++++ .../waf-ids-core/tests/dnsbl_zone_oracle.rs | 32 +++++ crates/waf-ids-core/tests/fuzz_invariants.rs | 29 ++++ .../waf-ids-core/tests/support/dnsbl_txt.rs | 4 + .../waf-ids-core/tests/support/dnsbl_zone.rs | 18 +++ docs/doctoring/dnsbl-txt-rdata-limits.md | 57 ++++++++ docs/fuzzing.md | 9 +- fuzz/fuzz_targets/fuzz_dnsbl_zone.rs | 1 + fuzz/support/dnsbl_zone.rs | 2 +- tests/dnsbl_rdlength_export.rs | 87 ++++++++++++ tests/support/dnsbl_txt.rs | 4 + 12 files changed, 436 insertions(+), 6 deletions(-) create mode 100644 crates/waf-ids-core/tests/dnsbl_rdlength.rs create mode 100644 docs/doctoring/dnsbl-txt-rdata-limits.md create mode 100644 tests/dnsbl_rdlength_export.rs diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index 1bfb7464..bab34f1a 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -394,6 +394,9 @@ pub fn validate_dnsbl(entry: &DnsblEntry) -> Result<(), &'static str> { if entry.source.trim().is_empty() { return Err("DNSBL source is required"); } + if !dnsbl_txt_fits_wire(entry) { + return Err("DNSBL TXT metadata exceeds 65535 wire bytes"); + } if entry.ttl_seconds == 0 { return Err("DNSBL ttl_seconds must be greater than 0"); } @@ -1419,7 +1422,10 @@ pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { let Ok(IpAddr::V4(code)) = IpAddr::from_str(&entry.code) else { continue; }; - if code.octets()[0] == 127 && (1..=DNSBL_MAX_TTL_SECONDS).contains(&entry.ttl_seconds) { + if code.octets()[0] == 127 + && (1..=DNSBL_MAX_TTL_SECONDS).contains(&entry.ttl_seconds) + && dnsbl_txt_fits_wire(entry) + { owner_ttls .entry(address) .and_modify(|ttl| *ttl = (*ttl).min(entry.ttl_seconds)) @@ -1427,9 +1433,12 @@ pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { } } for entry in entries { - // Persisted input bypasses admission; never publish a lifetime that is - // zero or outside RFC 2181's 31-bit wire range. - if entry.ttl_seconds == 0 || entry.ttl_seconds > DNSBL_MAX_TTL_SECONDS { + // Persisted input bypasses admission; omit invalid wire lifetimes and + // oversized metadata from both record and owner-lifetime projections. + if entry.ttl_seconds == 0 + || entry.ttl_seconds > DNSBL_MAX_TTL_SECONDS + || !dnsbl_txt_fits_wire(entry) + { continue; } if let IpAddr::V4(address) = entry.address { @@ -1499,6 +1508,32 @@ pub fn reverse_ipv4_for_dnsbl(octets: [u8; 4]) -> String { format!("{}.{}.{}.{}", octets[3], octets[2], octets[1], octets[0]) } +/// Check the complete TXT RDATA length without allocating an escaped copy. +/// Include one length octet for each UTF-8-safe character string and stop at +/// RFC 1035's unsigned 16-bit RDLENGTH ceiling. The separator is payload too. +fn dnsbl_txt_fits_wire(entry: &DnsblEntry) -> bool { + let mut wire_bytes = 1usize; + let mut chunk_bytes = 0usize; + for ch in entry + .reason + .chars() + .chain(" source=".chars()) + .chain(entry.source.chars()) + { + let bytes = ch.len_utf8(); + if chunk_bytes + bytes > 255 { + wire_bytes += 1; + chunk_bytes = 0; + } + wire_bytes += bytes; + if wire_bytes > 65_535 { + return false; + } + chunk_bytes += bytes; + } + true +} + /// Escape metadata and split it at UTF-8 character boundaries into adjacent /// TXT strings. Count decoded bytes, not master-file escape characters, so /// every string fits RFC 1035's one-octet length without dropping metadata. @@ -1538,6 +1573,28 @@ mod dnsbl_txt; mod tests { use super::*; + /// Exercise all new RDATA accounting branches in the unit-library object. + /// These are coverage controls; actual admission/export REDs are retained. + #[test] + fn dnsbl_unit_rdata_admission_and_publication_boundaries() { + let mut valid = AppData::seeded().dnsbl.remove(0); + valid.source = "unit".into(); + valid.reason = "๐Ÿ˜€".repeat(16_315); + valid.ttl_seconds = 600; + let mut invalid = valid.clone(); + invalid.reason.push('๐Ÿ˜€'); + invalid.ttl_seconds = 1; + assert_eq!(validate_dnsbl(&valid), Ok(())); + assert_eq!( + validate_dnsbl(&invalid), + Err("DNSBL TXT metadata exceeds 65535 wire bytes") + ); + let zone = export_dnsbl_zone("dnsbl.example", &[valid.clone(), invalid]); + assert_eq!(zone.lines().count(), 4); + assert!(zone.contains(" 600 IN A ")); + dnsbl_txt::assert_zone_txt_valid(&zone); + } + /// Exercise the validation matrix in the unit-test library instantiation, /// including existing non-TTL admission invariants and the new wire bound. #[test] diff --git a/crates/waf-ids-core/tests/dnsbl_rdlength.rs b/crates/waf-ids-core/tests/dnsbl_rdlength.rs new file mode 100644 index 00000000..4defb145 --- /dev/null +++ b/crates/waf-ids-core/tests/dnsbl_rdlength.rs @@ -0,0 +1,134 @@ +//! Aggregate TXT wire-size boundary without shrinking metadata silently. + +use waf_ids_core::{DnsblEntry, export_dnsbl_zone, validate_dnsbl}; + +#[path = "support/dnsbl_txt.rs"] +mod dnsbl_txt; + +fn ascii_entry(payload_bytes: usize) -> DnsblEntry { + DnsblEntry { + address: "192.0.2.10".parse().unwrap(), + code: "127.0.0.2".into(), + reason: "x".repeat(payload_bytes - 12), + source: "unit".into(), + ttl_seconds: 300, + prefix_len: None, + } +} + +/// Count actual UTF-8-safe chunk length octets independently in test code. +fn wire_bytes(entry: &DnsblEntry) -> usize { + let payload = format!("{} source={}", entry.reason, entry.source); + let mut chunks = vec![0usize]; + for scalar in payload.chars() { + let length = scalar.len_utf8(); + if chunks.last().unwrap() + length > 255 { + chunks.push(0); + } + *chunks.last_mut().unwrap() += length; + } + payload.len() + chunks.len() +} + +/// Stored metadata bypassing admission must not make a zone unloadable or +/// reduce a valid shared-owner RRset lifetime. Invalid evidence stays stored. +#[test] +fn oversized_persisted_metadata_is_omitted_from_both_publication_passes() { + let mut valid = ascii_entry(65_279); + valid.ttl_seconds = 600; + let mut invalid = ascii_entry(65_280); + invalid.ttl_seconds = 1; + let entries = [valid.clone(), invalid]; + let original = entries.clone(); + let zone = export_dnsbl_zone("dnsbl.example", &entries); + assert_eq!(entries, original); + dnsbl_txt::assert_zone_txt_valid(&zone); + assert_eq!( + zone.lines().count(), + 4, + "only the publishable entry remains" + ); + assert!(zone.contains("10.2.0.192 600 IN A 127.0.0.2\n")); + let text = zone + .lines() + .find_map(|line| line.split_once(" IN TXT ")) + .unwrap() + .1; + let chunks = dnsbl_txt::decode_txt_rdata(text); + assert_eq!( + chunks.iter().map(|chunk| 1 + chunk.len()).sum::(), + 65_535 + ); + assert_eq!( + chunks.concat(), + format!("{} source={}", valid.reason, valid.source).as_bytes() + ); + let reversed = [entries[1].clone(), entries[0].clone()]; + assert_eq!(export_dnsbl_zone("dnsbl.example", &reversed), zone); +} + +/// UTF-8 boundaries add length octets even when an ASCII payload of the same +/// byte length would fit. Escaped master-file spelling is not RDATA length. +#[test] +fn utf8_chunk_overhead_and_escaped_bytes_use_actual_wire_lengths() { + let mut valid = ascii_entry(12); + valid.reason = "๐Ÿ˜€".repeat(16_315); + let mut invalid = valid.clone(); + invalid.reason.push('๐Ÿ˜€'); + assert_eq!(wire_bytes(&valid), 65_532); + assert_eq!(wire_bytes(&invalid), 65_536); + assert_eq!(validate_dnsbl(&valid), Ok(())); + assert!(validate_dnsbl(&invalid).is_err()); + let zone = export_dnsbl_zone("dnsbl.example", &[valid.clone(), invalid]); + assert_eq!(zone.lines().count(), 4); + dnsbl_txt::assert_zone_txt_valid(&zone); + let decoded = dnsbl_txt::decode_txt_rdata( + zone.lines() + .last() + .unwrap() + .split_once(" IN TXT ") + .unwrap() + .1, + ) + .concat(); + assert_eq!( + decoded, + format!("{} source={}", valid.reason, valid.source).as_bytes() + ); + for scalar in ['"', '\\', '\n', '\0'] { + let mut escaped = ascii_entry(65_279); + escaped.reason = format!("x{}", scalar.to_string().repeat(65_266)); + assert_eq!(validate_dnsbl(&escaped), Ok(())); + let zone = export_dnsbl_zone("dnsbl.example", &[escaped.clone()]); + dnsbl_txt::assert_zone_txt_valid(&zone); + let text = zone + .lines() + .last() + .unwrap() + .split_once(" IN TXT ") + .unwrap() + .1; + let chunks = dnsbl_txt::decode_txt_rdata(text); + assert_eq!( + chunks.iter().map(|chunk| 1 + chunk.len()).sum::(), + 65_535 + ); + assert_eq!( + chunks.concat(), + format!("{} source={}", escaped.reason, escaped.source).as_bytes() + ); + } +} + +#[test] +fn admission_bounds_total_txt_rdata_including_chunk_length_octets() { + let valid = ascii_entry(65_279); + let invalid = ascii_entry(65_280); + assert_eq!(wire_bytes(&valid), 65_535); + assert_eq!(wire_bytes(&invalid), 65_536); + assert_eq!(validate_dnsbl(&valid), Ok(())); + assert_eq!( + validate_dnsbl(&invalid), + Err("DNSBL TXT metadata exceeds 65535 wire bytes") + ); +} diff --git a/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs b/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs index 8ec3f285..acc5fe55 100644 --- a/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs +++ b/crates/waf-ids-core/tests/dnsbl_zone_oracle.rs @@ -82,6 +82,38 @@ fn oracle_rejects_unparseable_origin_names() { } } +/// Accept a maximum-size TXT record, but reject aggregate RDATA overflow even +/// when every constituent character string remains legal. +#[test] +fn oracle_bounds_total_txt_rdata_and_omits_unpublishable_input() { + let mut entry = entries().remove(0); + entry.ttl_seconds = 300; + entry.reason = "x".repeat(65_267); + let full = format!("\"{}\"", "x".repeat(255)); + let valid_text = format!( + "{} \"{} source=unit\"", + vec![full.clone(); 255].join(" "), + "x".repeat(242) + ); + let valid_zone = format!( + "$ORIGIN dnsbl.example.\n$TTL 300\n10.2.0.192 IN A 127.0.0.2\n10.2.0.192 IN TXT {valid_text}\n" + ); + dnsbl_zone::assert_zone_matches_entries(&valid_zone, &[entry.clone()]); + let invalid_text = vec![full; 256].join(" "); + let invalid_zone = valid_zone.replace(&valid_text, &invalid_text); + assert!( + std::panic::catch_unwind(|| { + dnsbl_zone::dnsbl_txt::assert_zone_txt_valid(&invalid_zone); + }) + .is_err(), + "oracle accepted 65536 RDATA octets" + ); + let mut invalid = entry.clone(); + invalid.reason.push('x'); + invalid.ttl_seconds = 1; + dnsbl_zone::assert_zone_matches_entries(&valid_zone, &[entry, invalid]); +} + /// Reject malformed cache lifetimes while requiring valid controls to pass. #[test] fn oracle_rejects_wrong_or_inconsistent_cache_lifetimes() { diff --git a/crates/waf-ids-core/tests/fuzz_invariants.rs b/crates/waf-ids-core/tests/fuzz_invariants.rs index 60a79f88..762c7c2e 100644 --- a/crates/waf-ids-core/tests/fuzz_invariants.rs +++ b/crates/waf-ids-core/tests/fuzz_invariants.rs @@ -134,6 +134,34 @@ proptest! { prop_assert_eq!(entries, original, "export must not mutate source-owned entries"); } + // Force aggregate RDATA boundary cases as well as the short arbitrary-input + // path. A low-TTL oversized sibling must not influence the valid owner. + #[test] + fn dnsbl_rdata_boundary_preserves_valid_shared_owner_projection( + address in any::(), + payload_bytes in 65_270usize..65_290, + scalar in prop::sample::select(vec!['x', '๐Ÿ˜€', '\n', '"', '\\']), + ) { + let mut entry = DnsblEntry { + address: IpAddr::V4(Ipv4Addr::from(address)), + code: "127.0.0.2".into(), + reason: format!("x{}", scalar.to_string().repeat((payload_bytes - 13) / scalar.len_utf8())), + source: "unit".into(), + ttl_seconds: 1, + prefix_len: None, + }; + let mut valid = entry.clone(); + valid.reason = "short-positive".into(); + valid.ttl_seconds = 600; + // Metadata is nonblank, so admission must agree with the input oracle. + prop_assert_eq!(validate_dnsbl(&entry).is_ok(), dnsbl_zone::metadata_fits_rdata(&entry)); + let zone = export_dnsbl_zone("dnsbl.example", &[valid.clone(), entry.clone()]); + dnsbl_zone::assert_zone_matches_entries(&zone, &[valid.clone(), entry.clone()]); + entry.reason.push(scalar); + let zone = export_dnsbl_zone("dnsbl.example", &[entry.clone(), valid.clone()]); + dnsbl_zone::assert_zone_matches_entries(&zone, &[entry, valid]); + } + // DNSBL classification and zone generation must never panic; TXT payloads // must retain valid escaped strings and legal adjacent-string delimiters. #[test] @@ -171,6 +199,7 @@ proptest! { let txt_lines: Vec<_> = zone.lines().filter_map(|l| l.split_once(" IN TXT ")).collect(); let expected: Vec<_> = entries.iter().filter(|e| { (1..=2_147_483_647).contains(&e.ttl_seconds) + && dnsbl_zone::metadata_fits_rdata(e) && matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127) }).collect(); prop_assert_eq!(txt_lines.len(), expected.len()); diff --git a/crates/waf-ids-core/tests/support/dnsbl_txt.rs b/crates/waf-ids-core/tests/support/dnsbl_txt.rs index ac6bbd6a..e2ac01c8 100644 --- a/crates/waf-ids-core/tests/support/dnsbl_txt.rs +++ b/crates/waf-ids-core/tests/support/dnsbl_txt.rs @@ -59,6 +59,10 @@ pub fn assert_zone_txt_valid(zone: &str) { if let Some((_, text)) = line.split_once(" IN TXT ") { let strings = decode_txt_rdata(text); assert!(strings.iter().all(|string| string.len() <= 255)); + assert!( + strings.iter().map(|string| 1 + string.len()).sum::() <= 65_535, + "TXT RDATA exceeds the unsigned 16-bit RDLENGTH field" + ); } } } diff --git a/crates/waf-ids-core/tests/support/dnsbl_zone.rs b/crates/waf-ids-core/tests/support/dnsbl_zone.rs index 990b5550..73c7d2f4 100644 --- a/crates/waf-ids-core/tests/support/dnsbl_zone.rs +++ b/crates/waf-ids-core/tests/support/dnsbl_zone.rs @@ -6,6 +6,23 @@ pub mod dnsbl_txt; use std::net::IpAddr; use waf_ids_core::DnsblEntry; +/// Model UTF-8-safe chunking from input bytes without the production validator. +/// The independently allocated payload is bounded by the test/fuzz input budget. +pub fn metadata_fits_rdata(entry: &DnsblEntry) -> bool { + let payload = format!("{} source={}", entry.reason, entry.source); + let mut offset = 0; + let mut lengths = Vec::new(); + while offset < payload.len() { + let mut end = (offset + 255).min(payload.len()); + while !payload.is_char_boundary(end) { + end -= 1; + } + lengths.push(end - offset); + offset = end; + } + lengths.iter().map(|length| 1 + length).sum::() <= 65_535 +} + /// Check the exported records against input evidence without calling production /// validation, address reversal or lifetime projection helpers. Expected TTLs /// use an independent linear scan per owner rather than the exporter's map. @@ -15,6 +32,7 @@ pub fn assert_zone_matches_entries(zone: &str, entries: &[DnsblEntry]) { .iter() .filter(|entry| { entry.address.is_ipv4() + && metadata_fits_rdata(entry) && (1..=2_147_483_647).contains(&entry.ttl_seconds) && matches!(entry.code.parse::(), Ok(IpAddr::V4(code)) if code.octets()[0] == 127) }) diff --git a/docs/doctoring/dnsbl-txt-rdata-limits.md b/docs/doctoring/dnsbl-txt-rdata-limits.md new file mode 100644 index 00000000..f4c901e7 --- /dev/null +++ b/docs/doctoring/dnsbl-txt-rdata-limits.md @@ -0,0 +1,57 @@ +# DNSBL TXT aggregate RDATA limits + +## Root cause and reproduced boundary + +Per-string 255-byte chunking is necessary but not sufficient for a TXT RR. +Before this repair, admission accepted 65,280 ASCII payload bytes and the +exporter emitted 256 strings. The 256 length octets made RDATA 65,536 bytes. +The actual compiled Rust exporter fed dnspython 2.8.0: text loading and raw +RDATA serialization succeeded, but RR serialization failed with `FormError`. +The independently counted RDATA exceeded the unsigned 16-bit field. A 65,279-byte payload produced 65,535 RDATA bytes and +serialized successfully. The distinction matters: parsing text alone did not +prove that an RR could be encoded. + +## Primary contract and derived boundary + +Paul Mockapetris, RFC 1035, *Domain Names โ€” Implementation and Specification*, +November 1987, ยงยง3.2.1, 3.3, 3.3.14 and 4.1.3, +. +RDLENGTH is an unsigned 16-bit count of RDATA octets. TXT has one or more +character strings; each string has one length octet plus at most 255 data +bytes. Therefore the sum of decoded payload and all string-length octets +must be at most 65,535. This is a derived application boundary, not a new +DNS protocol limit or a complete DNS-message-size guarantee. + +## Minimal publication policy + +The payload is the existing `reason + " source=" + source`, in UTF-8. +Admission counts each scalar at the encoder's UTF-8-safe chunk boundaries, +without allocating an escaped copy, and stops at the wire ceiling. Oversized +metadata returns `DNSBL TXT metadata exceeds 65535 wire bytes`. +Escaped quotes, backslashes and controls count by decoded bytes, not their +longer master-file spelling. UTF-8 boundaries can use extra length octets: +16,316 emoji plus the separator/source occupies 65,536 wire bytes even +though an ASCII payload with the same decoded length could fit. + +Persisted input bypasses admission. An oversized entry is omitted from both +A/TXT publication and the shared-owner minimum-TTL projection. Its stored +evidence is not changed, truncated or clamped. Other valid entries retain +normal ordering, metadata and default/explicit TTL spelling. Existing IPv6, +prefix, response-code and feed-expiry policies are not expanded here. + +## Executed verification and limits + +Permanent tests use literal 65,535/65,536 controls, independent decoded TXT +lengths, UTF-8/escape cases and a valid shared owner alongside an oversized +low-TTL sibling. Real authenticated API admission/readback/export verifies +that rejection preserves the previous entry and exact zone bytes. Independent +oracle sensitivity includes otherwise-valid legal strings whose aggregate +RDATA exceeds the field. Stable boundary properties supplement short arbitrary +inputs; the fuzz oracle keeps its arbitrary-input path and uses independent +input-derived eligibility. + +Actual RR serialization does not establish UDP/TCP message fit, truncation, +authoritative DNS service deployment, traffic protection, a coverage-guided +fuzz campaign, whole-workspace 100% coverage or UI/locale acceptance. The +65,535-byte RDATA positive control itself can exceed a whole DNS message's +available payload after owner/header overhead; serving policy remains separate. diff --git a/docs/fuzzing.md b/docs/fuzzing.md index a4236abc..126c823c 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -46,7 +46,14 @@ TTL, missing-record and identity/metadata negatives. The same oracle checks ASCII origin labels of 1..=63 bytes and independently computes the encoded origin length, including label-length and root octets, with room for all four reversed IPv4 labels. Invalid origins use the existing `dnsbl.invalid` fallback; -ordinary origin spelling and record evidence remain unchanged. Oracle negatives +ordinary origin spelling and record evidence remain unchanged. TXT publication +also bounds total RDATA to 65,535 decoded payload-plus-length octets; oversized +persisted metadata is omitted from both record and owner-TTL projections. The +independent oracle computes UTF-8 chunk endpoints from input and counts emitted +length octets; stable boundary properties force near-limit metadata with a +short valid shared-owner control. See `doctoring/dnsbl-txt-rdata-limits.md` for +the actual admission/export/RR-serialization boundary and message-size limits. +Oracle negatives are test-sensitivity checks, not proof that a fuzz campaign ran. ## Running locally diff --git a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs index ac3961d1..0919359e 100644 --- a/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs +++ b/fuzz/fuzz_targets/fuzz_dnsbl_zone.rs @@ -140,6 +140,7 @@ fuzz_target!(|input: Input| { .iter() .filter(|e| { (1..=2_147_483_647).contains(&e.ttl_seconds) + && dnsbl_zone::metadata_fits_rdata(e) && matches!(e.code.parse::(), Ok(IpAddr::V4(ip)) if ip.octets()[0] == 127) }) .collect(); diff --git a/fuzz/support/dnsbl_zone.rs b/fuzz/support/dnsbl_zone.rs index 13f98137..e15433f7 100644 --- a/fuzz/support/dnsbl_zone.rs +++ b/fuzz/support/dnsbl_zone.rs @@ -3,4 +3,4 @@ #[path = "../../crates/waf-ids-core/tests/support/dnsbl_zone.rs"] mod oracle; -pub use oracle::{assert_zone_matches_entries, dnsbl_txt}; +pub use oracle::{assert_zone_matches_entries, dnsbl_txt, metadata_fits_rdata}; diff --git a/tests/dnsbl_rdlength_export.rs b/tests/dnsbl_rdlength_export.rs new file mode 100644 index 00000000..e07bc13a --- /dev/null +++ b/tests/dnsbl_rdlength_export.rs @@ -0,0 +1,87 @@ +//! Authenticated admission rejects oversized metadata without replacing evidence. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +#[tokio::test] +async fn rejected_txt_rdata_overflow_preserves_saved_entry_and_zone() { + let app = build_app(AppState::seeded(Some("rdlength-fixture-token".into()))); + let mut baseline_zone = None; + for (payload, status) in [ + (65_279, StatusCode::CREATED), + (65_280, StatusCode::BAD_REQUEST), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/dnsbl") + .header("content-type", "application/json") + .header("x-admin-token", "rdlength-fixture-token") + .body(Body::from( + serde_json::json!({ + "address": "192.0.2.10", "code": "127.0.0.2", + "reason": "x".repeat(payload - 12), "source": "unit", "ttl_seconds": 600 + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), status); + if status == StatusCode::BAD_REQUEST { + let body = to_bytes(response.into_body(), 4096).await.unwrap(); + assert!( + String::from_utf8(body.to_vec()) + .unwrap() + .contains("DNSBL TXT metadata exceeds 65535 wire bytes") + ); + } + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/api/dnsbl") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let entries: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()) + .unwrap(); + let entry = entries + .as_array() + .unwrap() + .iter() + .find(|entry| entry["address"] == "192.0.2.10") + .unwrap(); + assert_eq!(entry["reason"].as_str().unwrap().len(), 65_267); + assert_eq!(entry["source"], "unit"); + assert_eq!(entry["ttl_seconds"], 600); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/dnsbl/zone") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let zone = to_bytes(response.into_body(), 1_000_000).await.unwrap(); + if let Some(baseline) = &baseline_zone { + assert_eq!(&zone, baseline, "rejected write changed published bytes"); + } else { + baseline_zone = Some(zone); + } + } +} diff --git a/tests/support/dnsbl_txt.rs b/tests/support/dnsbl_txt.rs index ac6bbd6a..e2ac01c8 100644 --- a/tests/support/dnsbl_txt.rs +++ b/tests/support/dnsbl_txt.rs @@ -59,6 +59,10 @@ pub fn assert_zone_txt_valid(zone: &str) { if let Some((_, text)) = line.split_once(" IN TXT ") { let strings = decode_txt_rdata(text); assert!(strings.iter().all(|string| string.len() <= 255)); + assert!( + strings.iter().map(|string| 1 + string.len()).sum::() <= 65_535, + "TXT RDATA exceeds the unsigned 16-bit RDLENGTH field" + ); } } } From 1c855e93499d47f33496db4e94adbd8beb0ec8de Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 12:10:30 +0900 Subject: [PATCH 06/22] ci: route Wardnet workflows to isolated self-hosted runners --- .github/workflows/ci.yml | 5 +- .github/workflows/fuzz.yml | 5 +- .github/workflows/scorecard-analysis.yml | 3 +- tests/workflow_runner_contract.rs | 80 ++++++++++++++++-------- 4 files changed, 64 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3096352..be54524f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,12 @@ concurrency: jobs: rust: - runs-on: ubuntu-24.04 + # Require operator-provisioned isolated CI; never privileged control/model hosts. + runs-on: [self-hosted, Linux, X64, cwlab-ci-isolated] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable with: toolchain: stable diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 135966fd..668f5419 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -24,11 +24,14 @@ concurrency: jobs: fuzz: # One runner covers every bounded target; a matrix would consume four org slots. - runs-on: ubuntu-24.04 + # Require operator-provisioned isolated CI; never privileged control/model hosts. + runs-on: [self-hosted, Linux, X64, cwlab-ci-isolated] env: FUZZ_SECONDS: ${{ github.event_name == 'pull_request' && '60' || '300' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Install nightly toolchain uses: dtolnay/rust-toolchain@efcb852328a9f50117170cc43094fb6f09eaf1ae # nightly diff --git a/.github/workflows/scorecard-analysis.yml b/.github/workflows/scorecard-analysis.yml index 2d147bee..4e680023 100644 --- a/.github/workflows/scorecard-analysis.yml +++ b/.github/workflows/scorecard-analysis.yml @@ -13,7 +13,8 @@ permissions: jobs: analysis: name: Scorecard Analysis - runs-on: ubuntu-24.04 + # Keep the pinned analysis version; change execution placement only. + runs-on: [self-hosted, Linux, X64, cwlab-ci-isolated] permissions: contents: read security-events: write diff --git a/tests/workflow_runner_contract.rs b/tests/workflow_runner_contract.rs index ff42bad4..7fee9a57 100644 --- a/tests/workflow_runner_contract.rs +++ b/tests/workflow_runner_contract.rs @@ -1,17 +1,14 @@ -//! Repository contract for deterministic GitHub-hosted runner selection. +//! Repository contract for isolated self-hosted runner selection. //! -//! Wardnet's required pull-request workflows must not depend on GitHub's floating -//! `ubuntu-latest` alias. A floating image can change independently of the -//! repository and, during hosted-runner transitions, can leave exact-head jobs -//! queued before checkout. Pinning the Ubuntu image makes runner acquisition a -//! reviewed repository change while preserving GitHub-hosted execution. +//! Runner labels are routing requirements, not proof of isolation or capacity. +//! General CI executes repository code and must not select privileged control, +//! scanner or inference-host pools. Provisioning and actual job receipts remain +//! separate acceptance obligations. use std::fs; use std::path::Path; -const PINNED_UBUNTU_RUNNER: &str = "ubuntu-24.04"; -const FLOATING_UBUNTU_RUNNER: &str = "ubuntu-latest"; - +const ISOLATED_RUNNER: &str = "[self-hosted, Linux, X64, cwlab-ci-isolated]"; const RUNNER_BACKED_WORKFLOWS: &[&str] = &[ ".github/workflows/ci.yml", ".github/workflows/fuzz.yml", @@ -19,31 +16,62 @@ const RUNNER_BACKED_WORKFLOWS: &[&str] = &[ ]; #[test] -fn runner_backed_workflows_pin_the_hosted_ubuntu_image() { +fn every_local_workflow_requires_the_isolated_self_hosted_pool() { let repository = Path::new(env!("CARGO_MANIFEST_DIR")); - - for relative in RUNNER_BACKED_WORKFLOWS { - let path = repository.join(relative); - let workflow = fs::read_to_string(&path) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); - - assert!( - !workflow.contains(FLOATING_UBUNTU_RUNNER), - "{relative} must not use the floating {FLOATING_UBUNTU_RUNNER} runner alias" - ); - - let runners = workflow + let directory = repository.join(".github/workflows"); + let mut observed = Vec::new(); + for file in fs::read_dir(directory).unwrap() { + let path = file.unwrap().path(); + if !matches!( + path.extension().and_then(|ext| ext.to_str()), + Some("yml" | "yaml") + ) { + continue; + } + let workflow = fs::read_to_string(&path).unwrap(); + let runners: Vec<_> = workflow .lines() .filter_map(|line| line.trim().strip_prefix("runs-on:")) .map(str::trim) - .collect::>(); + .collect(); assert!( !runners.is_empty(), - "{relative} must define at least one runs-on value" + "{} must have a concrete runner", + path.display() ); assert!( - runners.iter().all(|runner| *runner == PINNED_UBUNTU_RUNNER), - "{relative} must use {PINNED_UBUNTU_RUNNER} for every runs-on value; found {runners:?}" + runners.iter().all(|runner| *runner == ISOLATED_RUNNER), + "{} must select only {ISOLATED_RUNNER}; found {runners:?}", + path.display() + ); + observed.push( + path.strip_prefix(repository) + .unwrap() + .to_str() + .unwrap() + .to_string(), + ); + } + observed.sort(); + assert_eq!( + observed, + RUNNER_BACKED_WORKFLOWS + .iter() + .map(|path| path.to_string()) + .collect::>() + ); +} + +#[test] +fn checkout_credentials_are_not_persisted_on_self_hosted_workers() { + let repository = Path::new(env!("CARGO_MANIFEST_DIR")); + for relative in RUNNER_BACKED_WORKFLOWS { + let workflow = fs::read_to_string(repository.join(relative)).unwrap(); + assert_eq!(workflow.matches("actions/checkout@").count(), 1); + assert_eq!( + workflow.matches("persist-credentials: false").count(), + 1, + "{relative} must not leave checkout credentials in the workspace" ); } } From 038842cf390ce0fbb85f2339cb2e983b4f28534e Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 18:56:17 +0900 Subject: [PATCH 07/22] fix(dnsbl): require publishable records for sale readiness --- crates/waf-ids-core/src/lib.rs | 38 +++-- .../tests/dnsbl_publication_readiness.rs | 150 +++++++++++++++++ docs/commercial/20b-krw-sale-readiness.md | 10 +- tests/dnsbl_publication_readiness.rs | 159 ++++++++++++++++++ 4 files changed, 340 insertions(+), 17 deletions(-) create mode 100644 crates/waf-ids-core/tests/dnsbl_publication_readiness.rs create mode 100644 tests/dnsbl_publication_readiness.rs diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index bab34f1a..e5b4fadb 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -1131,7 +1131,7 @@ pub fn commercial_readiness_snapshot_at(data: &AppData, now_unix: u64) -> Commer .iter() .any(|feed| !feed.stale && (feed.threat_count > 0 || feed.dnsbl_count > 0)); let route_ready = data.routes.iter().any(|route| route.enabled); - let dnsbl_ready = !data.dnsbl.is_empty(); + let dnsbl_ready = data.dnsbl.iter().any(dnsbl_entry_is_publishable); let support_evidence_ready = !data.events.is_empty(); let checks = vec![ @@ -1404,6 +1404,22 @@ pub fn readiness_check(id: &str, passed: bool, evidence: &str) -> ReadinessCheck } } +/// Whether a stored DNSBL entry can produce an A/TXT RR in the exported zone. +/// Kept in the domain crate so publication and commercial readiness share the +/// actual wire-admission contract instead of counting arbitrary stored rows. +fn dnsbl_entry_is_publishable(entry: &DnsblEntry) -> bool { + if !entry.address.is_ipv4() + || !(1..=DNSBL_MAX_TTL_SECONDS).contains(&entry.ttl_seconds) + || !dnsbl_txt_fits_wire(entry) + { + return false; + } + matches!( + IpAddr::from_str(&entry.code), + Ok(IpAddr::V4(code)) if code.octets()[0] == 127 + ) +} + /// Export IPv4 DNSBL entries with loopback answers and lossless TXT metadata. /// Each TXT character string is at most 255 decoded UTF-8 bytes (RFC 1035 /// sections 3.3 and 3.3.14); longer metadata uses adjacent quoted strings. @@ -1416,16 +1432,10 @@ pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { // without changing stored source evidence or depending on input order. let mut owner_ttls = std::collections::HashMap::::new(); for entry in entries { - let IpAddr::V4(address) = entry.address else { + if !dnsbl_entry_is_publishable(entry) { continue; - }; - let Ok(IpAddr::V4(code)) = IpAddr::from_str(&entry.code) else { - continue; - }; - if code.octets()[0] == 127 - && (1..=DNSBL_MAX_TTL_SECONDS).contains(&entry.ttl_seconds) - && dnsbl_txt_fits_wire(entry) - { + } + if let IpAddr::V4(address) = entry.address { owner_ttls .entry(address) .and_modify(|ttl| *ttl = (*ttl).min(entry.ttl_seconds)) @@ -1433,12 +1443,8 @@ pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { } } for entry in entries { - // Persisted input bypasses admission; omit invalid wire lifetimes and - // oversized metadata from both record and owner-lifetime projections. - if entry.ttl_seconds == 0 - || entry.ttl_seconds > DNSBL_MAX_TTL_SECONDS - || !dnsbl_txt_fits_wire(entry) - { + // Persisted input bypasses admission; omit entries that cannot be emitted. + if !dnsbl_entry_is_publishable(entry) { continue; } if let IpAddr::V4(address) = entry.address { diff --git a/crates/waf-ids-core/tests/dnsbl_publication_readiness.rs b/crates/waf-ids-core/tests/dnsbl_publication_readiness.rs new file mode 100644 index 00000000..fff790db --- /dev/null +++ b/crates/waf-ids-core/tests/dnsbl_publication_readiness.rs @@ -0,0 +1,150 @@ +//! Commercial publication readiness must describe emitted evidence, not storage. + +use waf_ids_core::{ + AppData, LicenseStatus, ReadinessStatus, SecurityEvent, ThreatFeedStatus, + buyer_evidence_manifest_at, commercial_readiness_snapshot_at, export_dnsbl_zone, +}; + +/// Keep every non-publication readiness criterion positive and deterministic. +fn otherwise_ready() -> AppData { + let mut data = AppData::seeded(); + data.commercial.license_status = LicenseStatus::Active; + data.commercial.license_id = Some("readiness-fixture".into()); + data.commercial.licensee = Some("fixture buyer".into()); + data.commercial.annual_contract_value_krw = Some(2_000_000_000); + data.threat_feeds.push(ThreatFeedStatus { + feed_id: "fixture-feed".into(), + source: "fixture".into(), + last_updated_unix: 10, + threat_count: 1, + dnsbl_count: 0, + ttl_seconds: 600, + }); + data.events.push(SecurityEvent { + id: 1, + timestamp_unix: 10, + client_ip: None, + route_id: Some("demo".into()), + action: "monitored".into(), + reason: "fixture".into(), + score: 0, + path: "/demo".into(), + }); + data +} + +#[test] +fn publication_readiness_rejects_nonempty_but_unpublishable_evidence() { + let positive = otherwise_ready(); + assert!(commercial_readiness_snapshot_at(&positive, 10).ready_for_enterprise_sale); + assert!(export_dnsbl_zone("dnsbl.example", &positive.dnsbl).contains(" IN A ")); + let seed = positive.dnsbl[0].clone(); + let mut cases = Vec::new(); + let mut entry = seed.clone(); + entry.address = "2001:db8::10".parse().unwrap(); + cases.push(("IPv6-only", entry)); + for ttl in [0, 2_147_483_648, u64::MAX] { + let mut entry = seed.clone(); + entry.ttl_seconds = ttl; + cases.push(("invalid TTL", entry)); + } + for code in ["8.8.8.8", "::1", "not-an-ip"] { + let mut entry = seed.clone(); + entry.code = code.into(); + cases.push(("invalid answer", entry)); + } + let mut entry = seed; + entry.reason = "x".repeat(65_280); + cases.push(("oversized TXT", entry)); + for (case, entry) in cases { + let mut data = positive.clone(); + data.dnsbl = vec![entry]; + let original = data.clone(); + assert_eq!( + export_dnsbl_zone("dnsbl.example", &data.dnsbl) + .lines() + .count(), + 2, + "{case}" + ); + let readiness = commercial_readiness_snapshot_at(&data, 10); + assert!( + !readiness.ready_for_enterprise_sale, + "{case} advertised sale readiness without emitted records" + ); + assert_eq!(readiness.blockers, ["dnsbl_publication"], "{case}"); + assert_eq!( + readiness + .checks + .iter() + .find(|check| check.id == "dnsbl_publication") + .unwrap() + .status, + ReadinessStatus::Fail + ); + let manifest = buyer_evidence_manifest_at(&data, 10); + assert!(!manifest.ready_for_enterprise_sale, "{case}"); + assert_eq!(manifest.blockers, ["dnsbl_publication"]); + assert_eq!( + manifest.runtime_counts.dnsbl_entry_count, 1, + "stored-evidence count is not published count" + ); + assert_eq!(data, original, "{case} mutated stored evidence"); + } +} + +#[test] +fn a_publishable_entry_clears_only_the_publication_blocker() { + let mut data = otherwise_ready(); + let valid = data.dnsbl[0].clone(); + data.dnsbl[0].address = "2001:db8::10".parse().unwrap(); + data.dnsbl.push(valid); + let original = data.clone(); + let readiness = commercial_readiness_snapshot_at(&data, 10); + assert!(readiness.ready_for_enterprise_sale); + assert!(readiness.blockers.is_empty()); + assert_eq!( + export_dnsbl_zone("dnsbl.example", &data.dnsbl) + .lines() + .count(), + 4 + ); + let manifest = buyer_evidence_manifest_at(&data, 10); + assert!(manifest.ready_for_enterprise_sale); + assert_eq!(manifest.runtime_counts.dnsbl_entry_count, 2); + data.commercial.annual_contract_value_krw = None; + assert_eq!( + commercial_readiness_snapshot_at(&data, 10).blockers, + ["contract_value"] + ); + assert_eq!(&data.dnsbl, &original.dnsbl); +} + +#[test] +fn publication_readiness_retains_exporter_boundary_and_legacy_metadata_behavior() { + let mut data = otherwise_ready(); + data.dnsbl[0].source = "unit".into(); + data.dnsbl[0].reason = "x".repeat(65_279 - " source=unit".len()); + data.dnsbl[0].ttl_seconds = 2_147_483_647; + let original = data.clone(); + assert!(commercial_readiness_snapshot_at(&data, 10).ready_for_enterprise_sale); + let zone = export_dnsbl_zone("dnsbl.example", &data.dnsbl); + assert_eq!(zone.lines().count(), 4); + assert!(zone.contains("2147483647 IN A ")); + assert_eq!(data, original); + + // Persisted legacy metadata need not satisfy create/import validation to + // remain safely exportable. Readiness is not a second admission policy. + for metadata in ["", " ", "\"\\\n\0", "ํ•œ๊ธ€๐Ÿ˜€"] { + data.dnsbl[0].reason = metadata.into(); + data.dnsbl[0].source = metadata.into(); + data.dnsbl[0].ttl_seconds = 300; + assert_eq!( + export_dnsbl_zone("dnsbl.example", &data.dnsbl) + .lines() + .count(), + 4 + ); + assert!(commercial_readiness_snapshot_at(&data, 10).ready_for_enterprise_sale); + } +} diff --git a/docs/commercial/20b-krw-sale-readiness.md b/docs/commercial/20b-krw-sale-readiness.md index 502188b4..be5dc4a5 100644 --- a/docs/commercial/20b-krw-sale-readiness.md +++ b/docs/commercial/20b-krw-sale-readiness.md @@ -46,7 +46,15 @@ This project treats a 2B KRW sale as an enterprise due-diligence threshold, not - `contract_value`: annual contract value is at least 2B KRW. - `threat_feed_updates`: at least one imported threat feed is fresh within its TTL. - `gateway_enforcement`: at least one enabled gateway route exists. -- `dnsbl_publication`: DNSBL entries are available for zone export. +- `dnsbl_publication`: at least one stored entry can actually emit an IPv4 + A/TXT record pair in `/dnsbl/zone`. Stored rows alone are not publication + evidence. IPv6-only rows, invalid loopback answers, zero/out-of-range TTLs, + and oversized TXT RDATA do not satisfy this check. + Readiness, the evidence manifest, and the support bundle use the same + publication predicate. Their DNSBL entry counts remain stored-evidence + counts; they are not published-record counts. Omitted entries remain stored + unchanged. A passing check does not prove authoritative DNS deployment, + resolver visibility, or enterprise release approval. - `support_evidence`: at least one security event exists for a support bundle. ## Current Boundary diff --git a/tests/dnsbl_publication_readiness.rs b/tests/dnsbl_publication_readiness.rs new file mode 100644 index 00000000..b13bd4e8 --- /dev/null +++ b/tests/dnsbl_publication_readiness.rs @@ -0,0 +1,159 @@ +//! Persisted DNSBL publication evidence must agree across all buyer-facing APIs. +//! These fixtures do not start a server, change credentials, or mutate live state. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::Value; +use std::{ + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppConfig, AppState, build_app}; +use waf_ids_core::{AppData, LicenseStatus, SecurityEvent, ThreatFeedStatus}; + +struct StateFixture(PathBuf); + +impl Drop for StateFixture { + fn drop(&mut self) { + // Only this test's unique disposable state file is owned here. + let _ = std::fs::remove_file(&self.0); + } +} + +async fn read(app: &axum::Router, path: &str) -> Vec { + let response = app + .clone() + .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + to_bytes(response.into_body(), 1_000_000) + .await + .unwrap() + .to_vec() +} + +fn assert_readiness(value: &Value, published: bool) { + assert_eq!(value["ready_for_enterprise_sale"], published); + assert_eq!( + value["readiness_level"], + if published { + "sale_ready" + } else { + "implementation_required" + } + ); + assert_eq!( + value["blockers"], + if published { + serde_json::json!([]) + } else { + serde_json::json!(["dnsbl_publication"]) + } + ); +} + +#[tokio::test] +async fn persisted_unpublishable_rows_do_not_advertise_sale_readiness() { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let mut data = AppData::seeded(); + data.commercial.license_status = LicenseStatus::Active; + data.commercial.license_id = Some("fixture-license".into()); + data.commercial.licensee = Some("fixture buyer".into()); + data.commercial.annual_contract_value_krw = Some(2_000_000_000); + data.threat_feeds.push(ThreatFeedStatus { + feed_id: "fixture-feed".into(), + source: "fixture".into(), + last_updated_unix: now, + threat_count: 1, + dnsbl_count: 0, + ttl_seconds: 600, + }); + data.events.push(SecurityEvent { + id: 1, + timestamp_unix: now, + client_ip: None, + route_id: Some("demo".into()), + action: "monitored".into(), + reason: "fixture".into(), + score: 0, + path: "/demo".into(), + }); + let valid = data.dnsbl[0].clone(); + let mut invalid_rows = Vec::new(); + let mut entry = valid.clone(); + entry.address = "2001:db8::10".parse().unwrap(); + invalid_rows.push(entry); + let mut entry = valid.clone(); + entry.ttl_seconds = 0; + invalid_rows.push(entry); + let mut entry = valid.clone(); + entry.ttl_seconds = 2_147_483_648; + invalid_rows.push(entry); + let mut entry = valid.clone(); + entry.code = "8.8.8.8".into(); + invalid_rows.push(entry); + let mut entry = valid.clone(); + entry.code = "127.0.0.2\nforged IN TXT \"extra\"".into(); + invalid_rows.push(entry); + let mut entry = valid.clone(); + entry.reason = "x".repeat(65_280); + invalid_rows.push(entry); + + for (case, invalid) in invalid_rows.into_iter().enumerate() { + for published in [false, true] { + data.dnsbl = vec![invalid.clone()]; + if published { + data.dnsbl.push(valid.clone()); + } + let fixture = StateFixture(std::env::temp_dir().join(format!( + "wardnet-publication-readiness-{}-{}-{case}-{published}.json", + std::process::id(), + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos() + ))); + let original = serde_json::to_vec_pretty(&data).unwrap(); + std::fs::write(&fixture.0, &original).unwrap(); + // Repeat a fresh load to exercise restart/read-only preservation. + for _ in 0..2 { + let mut config = AppConfig::memory(None); + config.state_path = Some(fixture.0.clone()); + let app = build_app(AppState::load(config).await.unwrap()); + let zone = String::from_utf8(read(&app, "/dnsbl/zone").await).unwrap(); + assert_eq!(zone.lines().count(), if published { 4 } else { 2 }); + let readiness: Value = + serde_json::from_slice(&read(&app, "/api/commercial/readiness").await).unwrap(); + assert_readiness(&readiness, published); + let manifest: Value = + serde_json::from_slice(&read(&app, "/api/commercial/evidence-manifest").await) + .unwrap(); + assert_readiness(&manifest, published); + assert_eq!( + manifest["runtime_counts"]["dnsbl_entry_count"], + data.dnsbl.len() + ); + let bundle: Value = + serde_json::from_slice(&read(&app, "/api/support-bundle").await).unwrap(); + assert_readiness(&bundle["readiness"], published); + assert_readiness(&bundle["evidence_manifest"], published); + assert_eq!(bundle["dnsbl_entry_count"], data.dnsbl.len()); + assert_eq!( + std::fs::read(&fixture.0).unwrap(), + original, + "read mutated persisted evidence" + ); + } + let fixture_path = fixture.0.clone(); + drop(fixture); + assert!( + !fixture_path.exists(), + "test-owned persisted fixture leaked" + ); + } + } +} From 570a3f27b6ac130832d2425d8826ee4d7467f73f Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 19:31:46 +0900 Subject: [PATCH 08/22] fix(smoke): download admin response before checking content --- docs/doctoring/smoke-admin-response.md | 63 ++++++++ scripts/smoke.sh | 5 +- tests/smoke_admin_response.rs | 206 +++++++++++++++++++++++++ 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 docs/doctoring/smoke-admin-response.md create mode 100644 tests/smoke_admin_response.rs diff --git a/docs/doctoring/smoke-admin-response.md b/docs/doctoring/smoke-admin-response.md new file mode 100644 index 00000000..b4f4c5d0 --- /dev/null +++ b/docs/doctoring/smoke-admin-response.md @@ -0,0 +1,63 @@ +# Complete admin-response smoke verification + +## Observed defect + +The distributed `scripts/smoke.sh` checked the admin page with a pipeline: +`curl ... | grep -q ...` under `set -euo pipefail`. A matching title near the +start of the page can make `grep -q` exit before curl finishes writing the +response. A valid HTTP 200 page can then fail the smoke check with curl exit 23. +The curl error reference identifies 23 as a local write/callback error [1]. + +`tests/smoke_admin_response.rs` retrieves the actual application `/admin` bytes +through Axum. A fixture-owned loopback server sends the title-bearing first +8,192 bytes, waits 150 ms, and sends the rest. The unmodified shipped shell +check failed with exit 23 and empty stderr. After the minimal repair, the same +complete response passes and the captured file equals every application byte. + +The earlier complete smoke execution also returned 23 with an empty log. +This controlled reproduction establishes a real defect in that consumer; +it does not retrospectively prove which command failed in the earlier run. +An initial test harness used a nonexistent `AppState::memory` constructor. +That compile error is retained separately and is not production RED evidence. + +## Repair and preserved failure behavior + +The harness downloads the complete response into its existing disposable +`TMP_DIR/admin.html`, then checks the title in that file. Curl must finish +successfully before grep runs. No HTTP or content validation is suppressed. +The existing temporary-directory cleanup also removes the downloaded page. + +The regression executes the actual extracted shell block, not a reconstructed +implementation. A fixture path containing spaces exercises shell quoting. +Alongside the application-byte positive, controls require: + +- complete HTTP 200 without the title: grep failure (1); +- HTTP 503 even with a title: curl HTTP failure (22); +- a title-bearing response shorter than its declared Content-Length: curl + partial-transfer failure (18). + +The meanings of curl 18 and 22 are documented in the same official reference +[1]. The test server, response framing, delays and error statuses are explicit +local fixtures, not observed deployed failures. Fixture deadlines bound this +regression; no global timeout, credential, provider, runner or protection +setting is changed. + +## Ownership and acceptance boundary + +A historical alternative that buffers the complete response is already +preserved in the mixed-context Draft PR #95. That PR explicitly prohibits +mechanical aggregate adoption. This repair does not import its Coraza, egress, +database or release changes, alter another checkout, or claim the workaround +was newly invented. It adds a minimal causal fix on this owner's current branch +and an executable regression for the actual consumer boundary. + +This check validates the admin page response and smoke consumer. It is not +pixel/accessibility/8-locale acceptance, an executed hosted job, protected merge, +release approval or a solution to runner capacity. Existing server processes +and original dirty files remain outside this repair's authority. + +## Reference + +[1] curl project. *libcurl error codes*: `CURLE_PARTIAL_FILE` (18), +`CURLE_HTTP_RETURNED_ERROR` (22), and `CURLE_WRITE_ERROR` (23). +https://curl.se/libcurl/c/libcurl-errors.html diff --git a/scripts/smoke.sh b/scripts/smoke.sh index d2a230a2..919aca72 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -88,7 +88,10 @@ assert_json_field "$health" 'data["event_limit"] == 5' assert_json_field "$health" 'data["admin_auth_configured"] is True' assert_json_field "$health" 'data["auth_mode"] == "production"' -curl -fsS "$BASE_URL/admin" | grep -q "ContextualWisdomLab WAF/IDS/AI SOC Gateway" +# Download the complete response before checking its content. With pipefail, +# grep -q can close the pipe after an early match and make curl exit 23. +curl -fsS "$BASE_URL/admin" -o "$TMP_DIR/admin.html" +grep -q "ContextualWisdomLab WAF/IDS/AI SOC Gateway" "$TMP_DIR/admin.html" unauthorized_code="$( curl -sS -o /dev/null -w '%{http_code}' \ diff --git a/tests/smoke_admin_response.rs b/tests/smoke_admin_response.rs new file mode 100644 index 00000000..0d822f14 --- /dev/null +++ b/tests/smoke_admin_response.rs @@ -0,0 +1,206 @@ +//! Execute the shipped smoke harness's admin-page check against real page bytes. +//! A fixture-owned loopback server controls packet timing without live services. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use std::{ + io::{Read, Write}, + net::TcpListener, + path::PathBuf, + process::{Command, Output, Stdio}, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +const TITLE: &str = "ContextualWisdomLab WAF/IDS/AI SOC Gateway"; + +fn actual_admin_check() -> &'static str { + let script = include_str!("../scripts/smoke.sh"); + let start = script.find("curl -fsS \"$BASE_URL/admin\"").unwrap(); + let end = script[start..].find("\n\nunauthorized_code=").unwrap() + start; + &script[start..end] +} + +struct FixtureDirectory(PathBuf); + +impl FixtureDirectory { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "wardnet smoke admin {} {}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir(&path).unwrap(); + Self(path) + } +} + +impl Drop for FixtureDirectory { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Replay a single response with a title-bearing first chunk and delayed tail. +/// The bytes are the application's actual /admin body, not a replacement page. +fn execute_check(body: Vec, status: u16, truncate: bool) -> (Output, FixtureDirectory) { + let directory = FixtureDirectory::new(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let address = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + let mut stream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!(Instant::now() < deadline, "fixture server accept timed out"); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("fixture accept failed: {error}"), + } + }; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0; 1024]; + let size = stream.read(&mut chunk).unwrap(); + assert!( + size > 0 && request.len() + size <= 4096, + "invalid fixture request framing" + ); + request.extend_from_slice(&chunk[..size]); + } + assert!(request.starts_with(b"GET /admin HTTP/1.1\r\n")); + let header = format!( + "HTTP/1.1 {status} Fixture\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(header.as_bytes()).unwrap(); + let split = 8192.min(body.len()); + stream.write_all(&body[..split]).unwrap(); + stream.flush().unwrap(); + thread::sleep(Duration::from_millis(150)); + // The pre-fix consumer may already have closed its output pipe. A + // socket error here is evidence of that early consumer, not a panic. + if !truncate { + let _ = stream.write_all(&body[split..]); + } + }); + let shell = format!("set -euo pipefail\n{}\n", actual_admin_check()); + let mut child = Command::new("bash") + .args(["-c", &shell]) + .env_clear() + .env("PATH", std::env::var_os("PATH").unwrap_or_default()) + .env("BASE_URL", format!("http://{address}")) + .env("TMP_DIR", &directory.0) + .env("NO_PROXY", "127.0.0.1") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(6); + let mut timed_out = false; + while child.try_wait().unwrap().is_none() { + if Instant::now() >= deadline { + child.kill().unwrap(); + timed_out = true; + break; + } + thread::sleep(Duration::from_millis(5)); + } + let output = child.wait_with_output().unwrap(); + server.join().unwrap(); + assert!(!timed_out, "smoke check exceeded owned diagnostic deadline"); + (output, directory) +} + +#[tokio::test] +async fn admin_smoke_check_accepts_complete_delayed_production_response() { + let response = build_app(AppState::seeded(None)) + .oneshot( + Request::builder() + .uri("/admin") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), 1_000_000) + .await + .unwrap() + .to_vec(); + assert!( + body.len() > 16_384, + "production page must exercise multiple curl writes" + ); + assert!(std::str::from_utf8(&body[..8192]).unwrap().contains(TITLE)); + let (output, directory) = execute_check(body.clone(), 200, false); + let captured = directory.0.join("admin.html"); + assert_eq!( + std::fs::read(captured).unwrap(), + body, + "smoke must retain the complete response" + ); + assert!( + output.status.success(), + "complete HTTP200 admin response failed shipped smoke check: exit={:?}, stderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn admin_smoke_check_rejects_complete_response_without_title() { + let (output, directory) = execute_check(vec![b'x'; 32_768], 200, false); + assert_eq!( + output.status.code(), + Some(1), + "missing title must fail grep" + ); + assert_eq!( + std::fs::read(directory.0.join("admin.html")).unwrap(), + vec![b'x'; 32_768] + ); +} + +#[test] +fn admin_smoke_check_rejects_http_error_even_with_a_title() { + let body = format!("{TITLE}{}", "x".repeat(32_768)).into_bytes(); + let (output, _) = execute_check(body, 503, false); + assert_eq!( + output.status.code(), + Some(22), + "curl HTTP error must propagate" + ); +} + +#[test] +fn admin_smoke_check_rejects_truncation_after_matching_title() { + let body = format!("{TITLE}{}", "x".repeat(32_768)).into_bytes(); + let (output, directory) = execute_check(body, 200, true); + assert_eq!( + output.status.code(), + Some(18), + "curl truncated response must propagate" + ); + assert_eq!( + std::fs::metadata(directory.0.join("admin.html")) + .unwrap() + .len(), + 8192 + ); +} From cdeec9fae0b3778d07440219cb998f647403bb05 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 20:03:08 +0900 Subject: [PATCH 09/22] fix(smoke): own the gateway process across restart and cleanup --- docs/doctoring/smoke-process-lifecycle.md | 65 +++++++++ scripts/smoke.sh | 23 ++- tests/smoke_process_lifecycle.rs | 165 ++++++++++++++++++++++ 3 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 docs/doctoring/smoke-process-lifecycle.md create mode 100644 tests/smoke_process_lifecycle.rs diff --git a/docs/doctoring/smoke-process-lifecycle.md b/docs/doctoring/smoke-process-lifecycle.md new file mode 100644 index 00000000..2e1b81ba --- /dev/null +++ b/docs/doctoring/smoke-process-lifecycle.md @@ -0,0 +1,65 @@ +# Smoke gateway process ownership + +## Observed defect + +The smoke script built the gateway and started `cargo run` in a background +subshell. `SERVER_PID` therefore identified the launcher rather than necessarily +the serving gateway. Its stop sequence signalled that PID, waited for it, and +cleared the PID before restart. The exact retained `start_server` and stop +sequence was executed with real Cargo and the real gateway on a fixture-owned +loopback listener. After stop, `/healthz` still responded. The diagnostic exited +99 to preserve that failure and then terminated only its newly created process +group. No legacy/shared server was stopped. + +This defect undermined restart evidence: the second startup could see the old +listener rather than a successfully restarted gateway. It also allowed a smoke +execution to leave its own gateway behind. Existing historical server PIDs are +not attributed to this defect without a matching producer identity. + +## Minimal repair + +Build the named gateway binary before the health wait, using Cargo's JSON +artifact stream. Select exactly one `compiler-artifact` with the gateway target +name and a non-null `executable` path. Cargo documents this path as the produced +executable, separately from the artifact filenames [1]. This avoids assuming +`target/debug`, ignoring `CARGO_TARGET_DIR`, or constructing a host binary path. + +The background shell uses `exec env ... "$SERVER_BIN"`, retaining the existing +bootstrap values and log redirection. `SERVER_PID` now owns the actual gateway. +The existing SIGTERM/wait stop sequence and EXIT trap reach that process. +Neither the gateway's shutdown implementation nor production configuration is +changed. A repeated exact real-Cargo probe confirms that the old listener is +unreachable after stop and its diagnostic process group is empty. + +The script retains `set -euo pipefail`: a failed build must stop startup even +if a partial artifact stream contains an executable. Missing, duplicate, +wrong-target and malformed artifact messages must also stop before execution. +The artifact parser is intentionally narrow and fails closed; arbitrary supplier +stdout is not silently accepted as an executable identity. + +## Retained regression and limits + +`tests/smoke_process_lifecycle.rs` executes the shipped shell prefix and real +Cargo-built gateway. During Cargo tests only artifact discovery is an explicit +synthetic command fixture, because recursively building the same target would +contend with the invoking Cargo lock. The complete smoke run and separate +RED/GREEN diagnostics retain actual Cargo execution as a distinct obligation. + +The lifecycle regression verifies direct-process reaping and listener closure, +a successful restart, and cleanup after a deliberate later shell failure. It +requires the second gateway PID to be absent and the temporary state directory +to be removed. Five offline artifact rejection controls require zero gateway +execution. Diagnostic timeouts and process-group cleanup apply only to newly +created fixture resources, never other owners or production services. + +Unix SIGTERM/reaping behavior is covered by the Unix regression. Other platforms, +hosted runner execution, arbitrary escaped descendants and a stalled graceful +shutdown are not certified by this test. Passing smoke is not protected merge, +release acceptance, complete coverage, or whole-product completion. Original +dirty files, legacy server processes and other worktrees remain outside scope. + +## Reference + +[1] The Rust Project. *The Cargo Book: External tools*, JSON messages and +artifact messages (`compiler-artifact`, `target`, `executable`). +https://doc.rust-lang.org/cargo/reference/external-tools.html diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 919aca72..5033765f 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -31,18 +31,33 @@ cleanup() { trap cleanup EXIT start_server() { - # Compile before the health wait so rustc time is not counted as a hang. - cargo build --quiet --manifest-path "$ROOT_DIR/Cargo.toml" + # Resolve the executable from Cargo's actual artifact rather than assuming a + # target directory. Exec it so SERVER_PID owns the gateway, not a cargo wrapper. + SERVER_BIN="$(cargo build --quiet --manifest-path "$ROOT_DIR/Cargo.toml" \ + --bin waf-ids-ai-soc --message-format=json | python3 -c ' +import json +import sys + +executables = [] +for line in sys.stdin: + artifact = json.loads(line) + if artifact.get("reason") == "compiler-artifact" and artifact.get("executable"): + if artifact.get("target", {}).get("name") == "waf-ids-ai-soc": + executables.append(artifact["executable"]) +if len(executables) != 1: + raise SystemExit("expected exactly one gateway executable from cargo build") +print(executables[0]) +')" ( cd "$ROOT_DIR" - BIND_ADDR="127.0.0.1:$PORT" \ + exec env BIND_ADDR="127.0.0.1:$PORT" \ ADMIN_TOKEN="$ADMIN_TOKEN_VALUE" \ ADMIN_TOKENS= \ WAF_IDS_CREDENTIALS_PATH= \ WAF_IDS_STATE_PATH="$STATE_FILE" \ DNSBL_ORIGIN="dnsbl.test" \ EVENT_LIMIT="5" \ - cargo run --quiet + "$SERVER_BIN" ) >"$LOG_FILE" 2>&1 & SERVER_PID="$!" diff --git a/tests/smoke_process_lifecycle.rs b/tests/smoke_process_lifecycle.rs new file mode 100644 index 00000000..9dba4e64 --- /dev/null +++ b/tests/smoke_process_lifecycle.rs @@ -0,0 +1,165 @@ +//! Exercise the shipped launch/stop/EXIT cleanup with the real gateway binary. +//! Cargo artifact discovery is an explicit offline fixture during Cargo tests: +//! recursively building the same target would contend for Cargo's own lock. + +#[test] +fn smoke_rejects_failed_or_ambiguous_cargo_artifact_discovery() { + use std::process::Command; + + let output = Command::new("python3") + .args([ + "-c", + r#" +import json, os, pathlib, subprocess, sys, tempfile +script_path, scratch = map(pathlib.Path, sys.argv[1:]) +script = script_path.read_text() +prefix = script.split('\nstart_server\n', 1)[0] +prefix = prefix.replace('ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"', 'ROOT_DIR="$OWNED_ROOT"') +with tempfile.TemporaryDirectory(prefix='wardnet cargo artifact ', dir=scratch) as directory: + root = pathlib.Path(directory) + tools = root / 'tools'; tools.mkdir() + sentinel = tools / 'gateway' + sentinel.write_text('#!/bin/sh\n: > "$EXECUTION_SENTINEL"\nexit 0\n'); sentinel.chmod(0o700) + artifact = {'reason': 'compiler-artifact', 'target': {'name': 'waf-ids-ai-soc'}, 'executable': str(sentinel)} + cases = [ + ('failed build with valid artifact', json.dumps(artifact), 42), + ('missing artifact', json.dumps({'reason': 'build-finished', 'success': True}), 0), + ('duplicate artifact', json.dumps(artifact) + '\n' + json.dumps(artifact), 0), + ('wrong target', json.dumps(dict(artifact, target={'name': 'unrelated'})), 0), + ('malformed JSON', '{broken', 0), + ] + for name, payload, exit_code in cases: + cargo = tools / 'cargo' + cargo.write_text('#!/bin/sh\nprintf "%s\\n" "$ARTIFACT_BYTES"\nexit "$ARTIFACT_EXIT"\n'); cargo.chmod(0o700) + env = {'PATH': str(tools) + os.pathsep + os.defpath + os.pathsep + '/opt/homebrew/bin', + 'TMPDIR': str(root), 'OWNED_ROOT': str(script_path.parent.parent), + 'ARTIFACT_BYTES': payload, 'ARTIFACT_EXIT': str(exit_code), + 'EXECUTION_SENTINEL': str(root / 'executed')} + driver = prefix + '\nstart_server\necho UNEXPECTED_START_SUCCESS\n' + result = subprocess.run(['bash', '-s'], input=driver.encode(), env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=6) + assert result.returncode != 0, (name, 'failed discovery passed') + assert not (root / 'executed').exists(), (name, 'binary executed before build acceptance') + assert b'UNEXPECTED_START_SUCCESS' not in result.stdout, name + print('five artifact rejection controls passed; no gateway executed') +"#, + concat!(env!("CARGO_MANIFEST_DIR"), "/scripts/smoke.sh"), + std::env::temp_dir().to_str().unwrap(), + ]) + .output() + .expect("run offline artifact-discovery controls"); + assert!( + output.status.success(), + "artifact discovery contract failed: stdout={}, stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +#[cfg(unix)] +fn smoke_owns_and_reaps_the_gateway_on_stop_restart_and_exit() { + use std::process::Command; + + let output = Command::new("python3") + .args([ + "-c", + r#" +import json, os, pathlib, signal, subprocess, sys, tempfile, time + +script_path, binary, scratch = map(pathlib.Path, sys.argv[1:]) +script = script_path.read_text() +prefix = script.split('\nstart_server\n', 1)[0] +prefix = prefix.replace('ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"', 'ROOT_DIR="$OWNED_ROOT"') +assert 'start_server()' in prefix and 'trap cleanup EXIT' in prefix + +with tempfile.TemporaryDirectory(prefix='wardnet smoke lifecycle ', dir=scratch) as fixture: + root = pathlib.Path(fixture) + tools = root / 'tools' + tools.mkdir() + cargo = tools / 'cargo' + cargo.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$OWNED_CARGO_ARGS"\nprintf "%s\\n" "$OWNED_CARGO_ARTIFACT"\n') + cargo.chmod(0o700) + env = { + 'PATH': str(tools) + os.pathsep + os.defpath + os.pathsep + '/opt/homebrew/bin', + 'TMPDIR': str(root), + 'OWNED_ROOT': str(script_path.parent.parent), + 'OWNED_CARGO_ARGS': str(root / 'cargo-args'), + 'OWNED_CARGO_ARTIFACT': json.dumps({'reason': 'compiler-artifact', 'target': {'name': 'waf-ids-ai-soc'}, 'executable': str(binary)}), + 'NO_PROXY': '127.0.0.1', + } + driver = prefix + ''' +start_server +FIRST_PID="$SERVER_PID" +printf 'FIRST_PID=%s\n' "$FIRST_PID" +kill "$SERVER_PID" +wait "$SERVER_PID" +SERVER_PID="" +if kill -0 "$FIRST_PID" 2>/dev/null; then exit 91; fi +if curl -fsS "$BASE_URL/healthz" >/dev/null 2>&1; then exit 92; fi +start_server +printf 'SECOND_PID=%s\n' "$SERVER_PID" +printf 'LISTENER=%s\n' "$BASE_URL" +printf 'STATE_ROOT=%s\n' "$TMP_DIR" +# Deliberately fail a later assertion: the EXIT trap must still stop the server. +exit 37 +''' + p = subprocess.Popen(['bash', '-s'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, start_new_session=True) + try: + stdout, stderr = p.communicate(driver.encode(), timeout=30) + markers = dict(line.split('=', 1) for line in stdout.decode().splitlines() if '=' in line) + assert p.returncode == 37, ('stop/restart did not settle', p.returncode, stderr.decode()) + assert 'FIRST_PID' in markers and 'SECOND_PID' in markers + for key in ['FIRST_PID', 'SECOND_PID']: + try: + os.kill(int(markers[key]), 0) + except ProcessLookupError: + pass + else: + raise AssertionError('owned gateway survived: ' + key) + health = subprocess.run(['curl', '-fsS', '--max-time', '2', markers['LISTENER'] + '/healthz'], + env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=4) + assert health.returncode != 0, 'EXIT cleanup left its listener serving' + assert not pathlib.Path(markers['STATE_ROOT']).exists(), 'EXIT cleanup left its owned state directory' + args = (root / 'cargo-args').read_text().splitlines() + assert args == ['build', '--quiet', '--manifest-path', str(script_path.parent.parent / 'Cargo.toml'), + '--bin', 'waf-ids-ai-soc', '--message-format=json'], args + print(json.dumps({'stop': 'reaped', 'restart': 'ready', 'exit_cleanup': 'reaped', + 'listener': 'unreachable', 'state_directory': 'removed', + 'cargo': 'fixture artifact discovery', 'binary': 'actual gateway'})) + finally: + # Scope is this newly created process group, never legacy/shared servers. + try: + os.killpg(p.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + p.wait(timeout=3) + except subprocess.TimeoutExpired: + os.killpg(p.pid, signal.SIGKILL) + p.wait(timeout=3) + deadline = time.monotonic() + 3 + while True: + try: + os.killpg(p.pid, 0) + except ProcessLookupError: + break + if time.monotonic() >= deadline: + os.killpg(p.pid, signal.SIGKILL) + raise AssertionError('diagnostic required forced descendant cleanup') + time.sleep(0.01) +"#, + concat!(env!("CARGO_MANIFEST_DIR"), "/scripts/smoke.sh"), + env!("CARGO_BIN_EXE_waf-ids-ai-soc"), + std::env::temp_dir().to_str().unwrap(), + ]) + .output() + .expect("run owned lifecycle fixture"); + assert!( + output.status.success(), + "smoke lifecycle contract failed: stdout={}, stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} From 5e8712aa5d9f8df7968bd87ee5e3d38abda4ff87 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 20:31:29 +0900 Subject: [PATCH 10/22] fix(tests): retain lifecycle child coverage profile routing --- docs/doctoring/smoke-process-lifecycle.md | 9 +++++++++ tests/smoke_process_lifecycle.rs | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/docs/doctoring/smoke-process-lifecycle.md b/docs/doctoring/smoke-process-lifecycle.md index 2e1b81ba..29b6ae58 100644 --- a/docs/doctoring/smoke-process-lifecycle.md +++ b/docs/doctoring/smoke-process-lifecycle.md @@ -52,6 +52,15 @@ to be removed. Five offline artifact rejection controls require zero gateway execution. Diagnostic timeouts and process-group cleanup apply only to newly created fixture resources, never other owners or production services. +Under instrumented test execution, the lifecycle fixture forwards only +`LLVM_PROFILE_FILE` from its parent into the otherwise explicit child-environment +allowlist. Without that route, the real gateway wrote default profiles outside +the collector and its two executions were omitted. The fixture still discards +ambient credentials and application configuration. A configured-route regression +and a full instrumented workspace replay verify the route; test success alone +is not proof of profile collection. Collection completeness does not raise or +waive the original 100% coverage requirement. + Unix SIGTERM/reaping behavior is covered by the Unix regression. Other platforms, hosted runner execution, arbitrary escaped descendants and a stalled graceful shutdown are not certified by this test. Passing smoke is not protected merge, diff --git a/tests/smoke_process_lifecycle.rs b/tests/smoke_process_lifecycle.rs index 9dba4e64..5919d1ad 100644 --- a/tests/smoke_process_lifecycle.rs +++ b/tests/smoke_process_lifecycle.rs @@ -88,6 +88,11 @@ with tempfile.TemporaryDirectory(prefix='wardnet smoke lifecycle ', dir=scratch) 'OWNED_CARGO_ARTIFACT': json.dumps({'reason': 'compiler-artifact', 'target': {'name': 'waf-ids-ai-soc'}, 'executable': str(binary)}), 'NO_PROXY': '127.0.0.1', } + if 'LLVM_PROFILE_FILE' in os.environ: + # Preserve only the profiler's output route, not ambient credentials or + # application configuration discarded by the fixture's allowlist. + env['LLVM_PROFILE_FILE'] = os.environ['LLVM_PROFILE_FILE'] + assert env.get('LLVM_PROFILE_FILE') == os.environ['LLVM_PROFILE_FILE'], 'lifecycle fixture dropped coverage output routing' driver = prefix + ''' start_server FIRST_PID="$SERVER_PID" From 6d6781298b99374a90cc155f2b526780b125d5a0 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 21:53:16 +0900 Subject: [PATCH 11/22] fix(opencti): retain all GraphQL file hashes --- docs/doctoring/opencti-graphql-file-hashes.md | 61 ++++++++++ src/opencti_import.rs | 108 +++++++++++++++++- tests/opencti_graphql_hashes.rs | 106 +++++++++++++++++ 3 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/opencti-graphql-file-hashes.md create mode 100644 tests/opencti_graphql_hashes.rs diff --git a/docs/doctoring/opencti-graphql-file-hashes.md b/docs/doctoring/opencti-graphql-file-hashes.md new file mode 100644 index 00000000..878cc349 --- /dev/null +++ b/docs/doctoring/opencti-graphql-file-hashes.md @@ -0,0 +1,61 @@ +# OpenCTI GraphQL file-hash import boundary + +## Observed defect + +The existing `/api/threat-intel/opencti` adapter accepted GraphQL connection +exports but interpreted `hashes` only as an algorithm-to-value JSON object. +The upstream GraphQL schema defines `StixFile` and `Artifact` hashes as an +array of `Hash` objects. Each object has an algorithm string and a nullable +hash string.[1] Consequently, a file with two hashes could silently import +only its hash-shaped `observable_value`, or be rejected when that display +value was a filename. The regression reproduced the two-hash input producing +one threat before the repair; a second fixture uses a filename display value. + +## Narrow repair + +For the existing file, StixFile and artifact mappings, the adapter now reads +explicit array entries from `algorithm` and `hash`. It trims both strings, +ignores absent, non-string or empty members, and preserves each usable pair +as a threat indicator with the existing lowercase convention, source, TTL +and severity. An explicit empty or unusable array does not fall back to the +display value. A file node with no usable pairs follows the existing skipped +object path. Mixed usable/unusable pairs retain usable evidence; this does +not introduce per-hash skipped counts. + +The old JSON object mapping and hash-shaped display fallback when there is +no array are unchanged. Algorithm names retain their existing spelling apart +from ASCII lowercase, including `SHA-256` becoming `sha-256`. This repair does +not define new hash algorithms, digest validation, scoring or malware policy. +It does not make all malformed non-array hash shapes fail closed. No file +observable recognition, STIX parser or generic threat validation is widened. + +## Regression evidence + +- Unit regression: two GraphQL hashes are retained with either a hash-like or + filename display value. Original single-hash RED and repaired GREEN are + retained outside the repository. +- Unit controls: null, missing, blank and wrongly typed pair fields, explicit + empty arrays, mixed usable/unusable pairs and existing object-map imports. +- Actual Axum consumer: unauthenticated POST is denied without changing rows; + authenticated GraphQL POST creates both threats, reports two imported + threats and no DNSBL entries, and readback preserves metadata. A subsequent + unusable-array POST is rejected without replacing rows or feed metadata. + +These are synthetic documents against real parser/HTTP code, not a live +OpenCTI pull or proof of upstream permissions, marking enforcement, deployed +persistence, complete STIX conformance, hosted CI, protected merge or release. +The OpenCTI import API remains an authenticated export-ingest adapter. + +## Sources + +[1] OpenCTI Platform. *OpenCTI GraphQL schema*, commit +`183acbc7f8f541f9af013722d41a1b95b320e314`, +`opencti-platform/opencti-graphql/config/schema/opencti.graphql`. +Sections `Hash`, `HashedObservable`, `Artifact` and `StixFile`. +Retrieved October 3, 2026 through the actual GitHub contents API and a +separate web retrieval. Immutable source: +https://raw.githubusercontent.com/OpenCTI-Platform/opencti/183acbc7f8f541f9af013722d41a1b95b320e314/opencti-platform/opencti-graphql/config/schema/opencti.graphql + +This is a schema compatibility repair, not a detection-model or routing-policy +feature. The authoritative schema is linked and summarized, not redistributed +as a claimed academic paper or new engine. diff --git a/src/opencti_import.rs b/src/opencti_import.rs index bcd70dee..2012f4f1 100644 --- a/src/opencti_import.rs +++ b/src/opencti_import.rs @@ -343,8 +343,36 @@ fn materialize_node(node: &serde_json::Value, source: &str, ttl_seconds: u64) -> }); } "file" | "stixfile" | "artifact" => { - // Prefer explicit hash fields when present. - if let Some(hashes) = node.get("hashes").and_then(|h| h.as_object()) { + // OpenCTI GraphQL uses [Hash { algorithm, hash }]; STIX/list + // exports may retain the existing algorithm-to-value object map. + // An explicit array is authoritative: never replace missing or + // malformed hash evidence with the observable's display value. + if let Some(hashes) = node.get("hashes").and_then(|h| h.as_array()) { + for item in hashes { + let algorithm = item + .get("algorithm") + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()); + let hash = item + .get("hash") + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()); + if let (Some(algorithm), Some(hash)) = (algorithm, hash) { + threats.push(ThreatIndicator { + value: hash.to_ascii_lowercase(), + indicator_type: algorithm.to_ascii_lowercase(), + severity: severity.clone(), + source: source.to_string(), + ttl_seconds, + }); + } + } + if threats.is_empty() { + return NodeOutcome::Skipped; + } + } else if let Some(hashes) = node.get("hashes").and_then(|h| h.as_object()) { let mut any = false; for (algo, hash_val) in hashes { if let Some(hash) = hash_val.as_str().map(str::trim).filter(|s| !s.is_empty()) { @@ -468,6 +496,82 @@ mod tests { use super::*; use std::net::Ipv4Addr; + /// GraphQL file hashes carry algorithm/hash pairs, not a STIX hash map. + #[test] + fn maps_every_graphql_file_hash_without_using_display_value() { + let md5 = "AB".repeat(16); + let sha256 = "CD".repeat(32); + for display in [serde_json::json!(md5), serde_json::json!("sample.exe")] { + let node = serde_json::json!({ + "entity_type": "StixFile", + "observable_value": display, + "hashes": [ + {"algorithm": "MD5", "hash": md5}, + {"algorithm": "SHA-256", "hash": sha256} + ], + "x_opencti_score": 80 + }); + let document = serde_json::json!({ + "data": {"stixCyberObservables": {"edges": [{"node": node}]}} + }); + let material = opencti_material_from_value(&document, "fixture", 600).unwrap(); + assert_eq!(material.threats.len(), 2, "display value lost a file hash"); + assert_eq!(material.threats[0].indicator_type, "md5"); + assert_eq!(material.threats[0].value, md5.to_ascii_lowercase()); + assert_eq!(material.threats[1].indicator_type, "sha-256"); + assert_eq!(material.threats[1].value, sha256.to_ascii_lowercase()); + assert!(material.threats.iter().all(|row| row.source == "fixture" + && row.ttl_seconds == 600 + && row.severity == Severity::Critical)); + assert!(material.dnsbl.is_empty()); + assert_eq!(material.skipped_objects, 0); + } + } + + /// Explicit GraphQL evidence must not fall back to a plausible display hash. + #[test] + fn skips_empty_or_malformed_graphql_hash_evidence() { + let display = "a".repeat(32); + for hashes in [ + serde_json::json!([]), + serde_json::json!([null, {}, {"algorithm": "MD5", "hash": null}, + {"algorithm": " ", "hash": display}, + {"algorithm": "MD5", "hash": " "}, + {"algorithm": 42, "hash": display}]), + ] { + let file = serde_json::json!({ + "entity_type": "StixFile", "observable_value": display, "hashes": hashes + }); + let valid = serde_json::json!({ + "entity_type": "Domain-Name", "observable_value": "fixture.example" + }); + let material = opencti_material_from_value( + &serde_json::json!({"entities": [file, valid]}), + "fixture", + 600, + ) + .unwrap(); + assert_eq!(material.threats.len(), 1); + assert_eq!(material.threats[0].indicator_type, "domain"); + assert_eq!(material.skipped_objects, 1); + } + let material = opencti_material_from_value( + &serde_json::json!({ + "entity_type": "Artifact", "observable_value": "file.bin", "hashes": [ + null, {"algorithm": " SHA-256 ", "hash": format!(" {} ", "AB".repeat(32))}, + {"algorithm": "MD5", "hash": " "} + ] + }), + "fixture", + 600, + ) + .unwrap(); + assert_eq!(material.threats.len(), 1); + assert_eq!(material.threats[0].indicator_type, "sha-256"); + assert_eq!(material.threats[0].value, "ab".repeat(32)); + assert_eq!(material.skipped_objects, 0); + } + #[test] fn maps_graphql_observables() { let raw = r#"{ diff --git a/tests/opencti_graphql_hashes.rs b/tests/opencti_graphql_hashes.rs new file mode 100644 index 00000000..4c1d0120 --- /dev/null +++ b/tests/opencti_graphql_hashes.rs @@ -0,0 +1,106 @@ +//! Actual Axum import/readback for OpenCTI GraphQL hash arrays. +//! Synthetic documents and credentials; no upstream service or live state. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +async fn read(app: &axum::Router, path: &str) -> Value { + let response = app + .clone() + .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap() +} + +async fn import(app: &axum::Router, document: &Value, token: Option<&str>) -> (StatusCode, Value) { + let mut request = Request::builder() + .method("POST") + .uri("/api/threat-intel/opencti?feed_id=graphql-hashes&source=fixture&ttl_seconds=600") + .header("content-type", "application/json"); + if let Some(token) = token { + request = request.header("x-admin-token", token); + } + let response = app + .clone() + .oneshot(request.body(Body::from(document.to_string())).unwrap()) + .await + .unwrap(); + let status = response.status(); + let body = to_bytes(response.into_body(), 1_000_000).await.unwrap(); + (status, serde_json::from_slice(&body).unwrap()) +} + +#[tokio::test] +async fn graphql_hash_array_import_preserves_all_hashes_and_authorization() { + // Test-owned value, not an operator credential or environment read. + let token = format!("opencti-test-{}", std::process::id()); + let app = build_app(AppState::seeded(Some(token.clone()))); + let md5 = "AB".repeat(16); + let sha256 = "CD".repeat(32); + let document = json!({"data": {"stixCyberObservables": {"edges": [{"node": { + "entity_type": "StixFile", "observable_value": "sample.exe", + "hashes": [{"algorithm": "MD5", "hash": md5}, + {"algorithm": "SHA-256", "hash": sha256}], + "x_opencti_score": 80 + }}]}}}); + let before = read(&app, "/api/threats").await; + let before_dnsbl = read(&app, "/api/dnsbl").await; + let before_feeds = read(&app, "/api/threat-feeds").await; + let (status, _) = import(&app, &document, None).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(read(&app, "/api/threats").await, before); + assert_eq!(read(&app, "/api/threat-feeds").await, before_feeds); + + let (status, result) = import(&app, &document, Some(&token)).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["upserted_threats"], 2); + assert_eq!(result["upserted_dnsbl"], 0); + assert_eq!(result["skipped_objects"], 0); + let rows = read(&app, "/api/threats").await; + let imported: Vec<_> = rows + .as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == "fixture") + .collect(); + assert_eq!(imported.len(), 2); + for (kind, value) in [ + ("md5", md5.to_ascii_lowercase()), + ("sha-256", sha256.to_ascii_lowercase()), + ] { + let row = imported + .iter() + .find(|row| row["indicator_type"] == kind) + .unwrap(); + assert_eq!(row["value"], value); + assert_eq!(row["ttl_seconds"], 600); + assert_eq!(row["severity"], "critical"); + } + assert_eq!(read(&app, "/api/dnsbl").await, before_dnsbl); + let feeds = read(&app, "/api/threat-feeds").await; + let feed = feeds + .as_array() + .unwrap() + .iter() + .find(|row| row["feed_id"] == "graphql-hashes") + .unwrap(); + assert_eq!(feed["threat_count"], 2); + assert_eq!(feed["dnsbl_count"], 0); + + let malformed = json!({"entities": [{ + "entity_type": "StixFile", "observable_value": "a".repeat(32), + "hashes": [{"algorithm": "MD5", "hash": null}] + }]}); + let (status, _) = import(&app, &malformed, Some(&token)).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(read(&app, "/api/threats").await, rows); + assert_eq!(read(&app, "/api/threat-feeds").await, feeds); + assert_eq!(read(&app, "/api/dnsbl").await, before_dnsbl); +} From d70166de8d6ee9d77e0e16e2bc028d673fd1c6bc Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 3 Oct 2026 22:19:00 +0900 Subject: [PATCH 12/22] fix(tests): wait for delayed admin fixture requests --- docs/doctoring/smoke-admin-response.md | 21 +++++++ tests/smoke_admin_response.rs | 81 +++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/smoke-admin-response.md b/docs/doctoring/smoke-admin-response.md index b4f4c5d0..f7135fb7 100644 --- a/docs/doctoring/smoke-admin-response.md +++ b/docs/doctoring/smoke-admin-response.md @@ -42,6 +42,27 @@ local fixtures, not observed deployed failures. Fixture deadlines bound this regression; no global timeout, credential, provider, runner or protection setting is changed. +## Accepted-socket fixture regression + +A subsequent committed-head workspace execution failed in the fixture's request +read with `WouldBlock`, before any response was sent. The fixture used a +nonblocking listener and assumed the accepted stream would block. An owned +native macOS probe reproduced an immediate `WouldBlock` when a client connected +before sending headers; explicitly switching the accepted stream to blocking +mode allowed the same delayed request to be read within the existing timeout. +This is a test-fixture defect, not a new gateway or curl response defect. + +The fixture now sets the accepted stream to blocking before applying its +unchanged two-second read/write timeouts. A test-owned loopback relay holds +request bytes for 150 ms after connecting, reproducing the original failure +without changing the extracted shipped curl check. The repaired test retains +complete-response success, truncated-response exit 18 and HTTP-error exit 22, +and joins the relay and server threads before reporting failure. Existing +accept and child deadlines remain in place; no global timeout is widened. +The initial committed-head failure and deterministic delayed-header RED are +retained separately from GREEN. No production shell or gateway change is +part of this fixture repair. + ## Ownership and acceptance boundary A historical alternative that buffers the complete response is already diff --git a/tests/smoke_admin_response.rs b/tests/smoke_admin_response.rs index 0d822f14..33ea7389 100644 --- a/tests/smoke_admin_response.rs +++ b/tests/smoke_admin_response.rs @@ -51,6 +51,15 @@ impl Drop for FixtureDirectory { /// Replay a single response with a title-bearing first chunk and delayed tail. /// The bytes are the application's actual /admin body, not a replacement page. fn execute_check(body: Vec, status: u16, truncate: bool) -> (Output, FixtureDirectory) { + execute_check_with_request_delay(body, status, truncate, false) +} + +fn execute_check_with_request_delay( + body: Vec, + status: u16, + truncate: bool, + delayed_request: bool, +) -> (Output, FixtureDirectory) { let directory = FixtureDirectory::new(); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); listener.set_nonblocking(true).unwrap(); @@ -67,6 +76,9 @@ fn execute_check(body: Vec, status: u16, truncate: bool) -> (Output, Fixture Err(error) => panic!("fixture accept failed: {error}"), } }; + // macOS can retain the listener's nonblocking mode on accepted sockets. + // Acceptance does not mean HTTP bytes arrived; timed reads must wait. + stream.set_nonblocking(false).unwrap(); stream .set_read_timeout(Some(Duration::from_secs(2))) .unwrap(); @@ -99,12 +111,63 @@ fn execute_check(body: Vec, status: u16, truncate: bool) -> (Output, Fixture let _ = stream.write_all(&body[split..]); } }); + // A test-owned relay connects before forwarding the HTTP header, making + // the accepted-socket read mode observable without changing the curl check. + let mut relay = None; + let check_address = if delayed_request { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let relay_address = listener.local_addr().unwrap(); + relay = Some(thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(5); + let mut downstream = loop { + match listener.accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!(Instant::now() < deadline, "relay accept timed out"); + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("relay accept: {error}"), + } + }; + downstream.set_nonblocking(false).unwrap(); + downstream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + downstream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut upstream = std::net::TcpStream::connect(address).unwrap(); + upstream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + upstream + .set_write_timeout(Some(Duration::from_secs(2))) + .unwrap(); + // The fixture polls accept every 5ms; no request bytes are sent + // until well after it accepts the new connection. + thread::sleep(Duration::from_millis(150)); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0; 1024]; + let size = downstream.read(&mut chunk).unwrap(); + assert!(size > 0 && request.len() + size <= 4096); + request.extend_from_slice(&chunk[..size]); + } + if upstream.write_all(&request).is_ok() { + let _ = std::io::copy(&mut upstream, &mut downstream); + } + })); + relay_address + } else { + address + }; let shell = format!("set -euo pipefail\n{}\n", actual_admin_check()); let mut child = Command::new("bash") .args(["-c", &shell]) .env_clear() .env("PATH", std::env::var_os("PATH").unwrap_or_default()) - .env("BASE_URL", format!("http://{address}")) + .env("BASE_URL", format!("http://{check_address}")) .env("TMP_DIR", &directory.0) .env("NO_PROXY", "127.0.0.1") .stdout(Stdio::null()) @@ -122,7 +185,11 @@ fn execute_check(body: Vec, status: u16, truncate: bool) -> (Output, Fixture thread::sleep(Duration::from_millis(5)); } let output = child.wait_with_output().unwrap(); - server.join().unwrap(); + let server_result = server.join(); + if let Some(relay) = relay { + relay.join().unwrap(); + } + server_result.unwrap(); assert!(!timed_out, "smoke check exceeded owned diagnostic deadline"); (output, directory) } @@ -163,6 +230,16 @@ async fn admin_smoke_check_accepts_complete_delayed_production_response() { ); } +/// TCP connection establishment does not imply that HTTP header bytes arrived. +#[test] +fn admin_smoke_check_waits_for_delayed_request_headers() { + let body = format!("{TITLE}{}", "x".repeat(32_768)).into_bytes(); + for (status, truncate, expected) in [(200, false, 0), (200, true, 18), (503, false, 22)] { + let (output, _) = execute_check_with_request_delay(body.clone(), status, truncate, true); + assert_eq!(output.status.code(), Some(expected)); + } +} + #[test] fn admin_smoke_check_rejects_complete_response_without_title() { let (output, directory) = execute_check(vec![b'x'; 32_768], 200, false); From c22fc0287062d8e329d332c1f1ecb74efc1c871a Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sun, 4 Oct 2026 00:02:37 +0900 Subject: [PATCH 13/22] fix(ingest): truncate engine reasons at UTF-8 boundaries --- docs/doctoring/engine-reason-utf8.md | 59 ++++++++++++ src/lib.rs | 71 +++++++++++++- tests/engine_reason_utf8.rs | 133 +++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 docs/doctoring/engine-reason-utf8.md create mode 100644 tests/engine_reason_utf8.rs diff --git a/docs/doctoring/engine-reason-utf8.md b/docs/doctoring/engine-reason-utf8.md new file mode 100644 index 00000000..95ddc032 --- /dev/null +++ b/docs/doctoring/engine-reason-utf8.md @@ -0,0 +1,59 @@ +# UTF-8-safe engine enforcement reasons + +## Observed product defect + +The existing authenticated Coraza audit and Suricata EVE ingest routes share +`apply_engine_enforcement_hints`. For a block-grade hit it derived a shorter +DNSBL reason from the full engine event. When the trimmed reason exceeded 200 +bytes, it sliced at byte 199 and appended an ellipsis. A long Korean reason +caused both actual Axum import routes to panic at that slice before completing +the import. The retained final RED contains two product failures at the same +source line. Earlier fixture preassertion failures are separate and excluded +from product RED acceptance. + +Rust string offsets count bytes. A string slice endpoint must coincide with a +UTF-8 character boundary; otherwise indexing panics. `is_char_boundary` checks +that boundary.[1] Engine messages are valid UTF-8 text but need not be ASCII. + +## Narrow repair and unchanged policy + +For trimmed input longer than 200 bytes, move the existing 199-byte cutoff +backward to the closest character boundary, then retain the existing ellipsis. +For valid UTF-8 this requires at most three one-byte steps. ASCII output stays +byte-identical. No new dependency, unsafe conversion or language-specific +message filtering is introduced. + +The existing threshold, trim behavior, short-message behavior and blank-message +fallback are unchanged. The budget remains at most 199 prefix bytes followed +by the three-byte UTF-8 ellipsis: up to 202 bytes, not a newly imposed 200-byte +cap. This truncates scalar values safely; it is not grapheme-cluster-aware. +Full event reasons are retained unchanged. Source tags, scores, block/monitor +policy, DNSBL answer codes, 3600-second hint TTL, path selection, authorization +and persistence transaction logic are unchanged. + +## Verification boundary + +- Actual Axum Coraza and Suricata import regressions use synthetic Korean and + emoji messages, require CREATED, full event readback, exact bounded DNSBL + reason, hint counts, TTL/code preservation and unauthorized nonmutation. +- Unit controls assert independent literal results at ASCII 199/200/201 bytes, + two-, three- and four-byte scalar cutoffs, a valid cutoff before a multibyte + tail, whitespace trimming, blank fallback and low-score monitor nonmutation. +- The sibling engine call paths share the same repaired projection; the engine + adapters are not reimplemented or replaced with synthetic detections. + +These fixtures execute real parser/router/domain code; they do not run live +Coraza/CRS or Suricata, deploy a gateway, prove all failure rollback paths, +complete 100% coverage, hosted review, protected merge or release acceptance. +The previously preserved runtime processes and other owners are outside scope. + +## Reference + +[1] Rust project. *Primitive type str*: `len`, `is_char_boundary`, and +`SliceIndex for RangeTo` panic conditions. Official standard library +API, retrieved October 3, 2026 through actual web open/find results. +https://doc.rust-lang.org/std/primitive.str.html#method.is_char_boundary + +This is a string-boundary correctness repair, not a detection-model, scoring +or routing-policy feature. The official language contract is cited rather +than attaching unrelated detection research. diff --git a/src/lib.rs b/src/lib.rs index 38c1559f..532ab82e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1942,7 +1942,12 @@ fn apply_engine_enforcement_hints( let reason = { let trimmed = reason.trim(); if trimmed.len() > 200 { - format!("{}โ€ฆ", &trimmed[..199]) + // Keep the existing byte allowance, but never split a UTF-8 scalar. + let mut end = 199; + while !trimmed.is_char_boundary(end) { + end -= 1; + } + format!("{}โ€ฆ", &trimmed[..end]) } else if trimmed.is_empty() { format!("{source} engine hit") } else { @@ -5926,6 +5931,70 @@ mod tests { ); } + /// Keep the original byte budget and fallback without UTF-8 slicing panics. + #[test] + fn engine_hint_reason_boundary_and_policy_controls() { + let ip = "192.0.2.201".parse().unwrap(); + let cases = [ + ("x".repeat(199), "x".repeat(199)), + ("x".repeat(200), "x".repeat(200)), + ("x".repeat(201), format!("{}โ€ฆ", "x".repeat(199))), + ("รฉ".repeat(110), format!("{}โ€ฆ", "รฉ".repeat(99))), + ("ํ•œ".repeat(80), format!("{}โ€ฆ", "ํ•œ".repeat(66))), + ("๐Ÿ›ก".repeat(60), format!("{}โ€ฆ", "๐Ÿ›ก".repeat(49))), + ( + format!("{}รฉ", "x".repeat(199)), + format!("{}โ€ฆ", "x".repeat(199)), + ), + (" short ํ•œ ".into(), "short ํ•œ".into()), + (" \n\t ".into(), "fixture engine hit".into()), + ]; + for (input, expected) in cases { + let mut data = AppData::seeded(); + assert_eq!( + apply_engine_enforcement_hints( + &mut data, + "fixture", + "block", + Some(ip), + "/case?x=1", + &input, + 80 + ), + 3 + ); + let entry = data + .dnsbl + .iter() + .find(|row| row.source == "fixture") + .unwrap(); + assert_eq!(entry.reason, expected); + assert_eq!(entry.ttl_seconds, 3_600); + assert_eq!(entry.code, "127.0.0.2"); + assert!( + entry.reason.len() <= 202, + "199 bytes plus three-byte ellipsis" + ); + } + let mut data = AppData::seeded(); + let original_dnsbl = data.dnsbl.clone(); + let original_threats = data.threats.clone(); + assert_eq!( + apply_engine_enforcement_hints( + &mut data, + "fixture", + "monitor", + Some(ip), + "/case", + &"ํ•œ".repeat(80), + 25 + ), + 0 + ); + assert_eq!(data.dnsbl, original_dnsbl); + assert_eq!(data.threats, original_threats); + } + #[tokio::test] async fn suricata_eve_ingest_maps_alerts_to_security_events() { let app = build_app(AppState::seeded(Some("secret".to_string()))); diff --git a/tests/engine_reason_utf8.rs b/tests/engine_reason_utf8.rs new file mode 100644 index 00000000..0cc088f3 --- /dev/null +++ b/tests/engine_reason_utf8.rs @@ -0,0 +1,133 @@ +//! Real management routes must not panic while deriving bounded engine hints. +//! Documents and credentials are synthetic; no live engine/service is used. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +async fn read(app: &axum::Router, path: &str) -> Value { + let response = app + .clone() + .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap() +} + +async fn check_engine(coraza: bool) { + let token = format!("engine-utf8-fixture-{}", std::process::id()); + let (endpoint, source, prefix) = if coraza { + ("/api/waf/coraza/audit", "engine:coraza", "coraza/crs: ") + } else { + ("/api/ids/suricata/eve", "engine:suricata", "suricata: ") + }; + for message in [ + format!("{}{}", if coraza { "" } else { "x" }, "ํ•œ".repeat(90)), + "๐Ÿ›ก".repeat(70), + ] { + let app = build_app(AppState::seeded(Some(token.clone()))); + let expected_full = format!("{prefix}{message}"); + assert!(expected_full.len() > 200); + assert!( + !expected_full.is_char_boundary(199), + "fixture must bisect UTF-8" + ); + let document = if coraza { + json!({"transaction": { + "client_ip": "192.0.2.201", "is_interrupted": true, + "request": {"uri": "/utf8?fixture=1"} + }, "messages": [{"message": message, "data": {"severity": 1}}]}) + } else { + json!({"event_type": "alert", "src_ip": "192.0.2.201", + "http": {"url": "/utf8?fixture=1"}, + "alert": {"signature": message, "severity": 1}}) + }; + let before = read(&app, "/api/dnsbl").await; + let unauthorized = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(endpoint) + .header("content-type", "application/json") + .body(Body::from(document.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unauthorized.status(), StatusCode::UNAUTHORIZED); + assert_eq!(read(&app, "/api/dnsbl").await, before); + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(endpoint) + .header("content-type", "application/json") + .header("x-admin-token", &token) + .body(Body::from(document.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let result: Value = + serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap(); + assert_eq!(result["enforcement_hints"], 3); + let events = read(&app, "/api/events").await; + assert!( + events + .as_array() + .unwrap() + .iter() + .any(|row| row["reason"] == expected_full + && row["action"] == "block" + && row["score"] == 80) + ); + // Independent expected prefix: whole scalar values whose cumulative + // UTF-8 length fits the existing 199-byte payload allowance. + let mut expected = String::new(); + for ch in expected_full.chars() { + if expected.len() + ch.len_utf8() > 199 { + break; + } + expected.push(ch); + } + expected.push('โ€ฆ'); + let rows = read(&app, "/api/dnsbl").await; + let row = rows + .as_array() + .unwrap() + .iter() + .find(|row| row["source"] == source) + .unwrap(); + assert_eq!(row["reason"], expected); + assert_eq!(row["ttl_seconds"], 3600); + assert_eq!(row["code"], "127.0.0.2"); + let threats = read(&app, "/api/threats").await; + assert_eq!( + threats + .as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == source) + .count(), + 2 + ); + } +} + +#[tokio::test] +async fn coraza_long_unicode_reason_preserves_events_and_bounded_hints() { + check_engine(true).await; +} + +#[tokio::test] +async fn suricata_long_unicode_reason_preserves_events_and_bounded_hints() { + check_engine(false).await; +} From b00ed6915f1bf3cc37d452f70b41c290c4e167a5 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sun, 4 Oct 2026 01:37:54 +0900 Subject: [PATCH 14/22] fix(ingest): honor numeric Suricata timestamp offsets --- docs/doctoring/suricata-timestamp-offset.md | 84 +++++++++++++++ src/suricata_eve.rs | 113 ++++++++++++++++++-- tests/suricata_timestamp_offset.rs | 103 ++++++++++++++++++ 3 files changed, 291 insertions(+), 9 deletions(-) create mode 100644 docs/doctoring/suricata-timestamp-offset.md create mode 100644 tests/suricata_timestamp_offset.rs diff --git a/docs/doctoring/suricata-timestamp-offset.md b/docs/doctoring/suricata-timestamp-offset.md new file mode 100644 index 00000000..40bf74e6 --- /dev/null +++ b/docs/doctoring/suricata-timestamp-offset.md @@ -0,0 +1,84 @@ +# Suricata event occurrence time and numeric UTC offsets + +## Observed defect + +The EVE adapter parsed only the first 19 timestamp characters. It interpreted +local wall time as UTC regardless of the numeric offset in the suffix. The +actual authenticated `/api/ids/suricata/eve` route stored two records for the +same instant with different Unix seconds: the `+0100` record was 3600 seconds +late and the `-0530` record was 19800 seconds early. Both represent UTC +`2024-06-15T12:34:56Z`, or Unix second 1718454896. + +Official Suricata EVE documentation includes numeric offsets such as `+0100` +in its alert examples.[1] Numeric offsets express local time minus UTC, so +conversion to UTC subtracts the signed offset.[2] Treating the suffix as inert +text corrupts event chronology. The retained unit RED also demonstrates that +an invalid `+2400` suffix was silently accepted as UTC. + +An earlier integration-test compilation error tried importing a private +module. Another fixture failed on an unauthenticated audit-log read. Both are +harness failures, not product RED. The corrected unit tests and authenticated +route produced the timestamp assertion failures before production edits. + +## Narrow repair + +Keep the existing date/time prefix parser, second-level precision, whitespace +trimming and timezone-less lab interpretation. Parse an optional nonempty +ASCII fractional-seconds component, then require either no timezone, `Z`, or +a signed `HHMM`/`HH:MM` numeric offset. Require offset hours at most 23 and +minutes at most 59, and consume the entire suffix. Subtract the offset using +signed checked arithmetic before converting to nonnegative Unix seconds. +This also allows a pre-epoch local wall time whose negative offset places the +actual UTC instant at or after the epoch. + +Malformed suffixes and UTC instants before the epoch return `None`. The +existing route then uses its existing ingest-time fallback; it does not drop +the alert or change HTTP admission. No dependency or public API is added. +The private adapter module stays private. Scores, alert actions, enforcement +hints, original reason text, source tags, DNSBL codes/TTLs, authorization and +persistence transaction boundaries are unchanged. + +This is not full RFC 3339 validation. Calendar-day validity and leap-second +handling retain the existing prefix parser's limitations. Fractional seconds +are validated but discarded because the existing event model stores seconds. +Timezone-less timestamps remain a lab compatibility convention, not an +inferred local timezone. The existing zero-offset instant handling is retained; +no extra timezone provenance is introduced into the event model. + +## Verification + +- Unit controls compare `Z`, positive/negative compact and colon offsets to an + independent literal Unix instant, including fractional seconds, epoch + crossings and a negative UTC result. +- Malformed suffix controls include oversized hours/minutes, incomplete or + nonnumeric offsets, empty/nonnumeric fractions, trailing garbage and Unicode. + Existing timezone-less and zero-offset forms remain positive controls. +- An actual authenticated Axum array import must retain identical occurrence + times for positive and negative offsets, original policy scores/actions, + hint counts, DNSBL code/TTL and unauthorized event/threat/DNSBL/audit + nonmutation. +- Full workspace tests, formatting, warning-fatal Clippy and the real external + gateway smoke are separate parent-run gates. A former passing source review + does not approve this new delta; the complete original-base union needs a + fresh source-bound independent verdict. + +Fixtures are offline synthetic EVE records through actual Rust parser/router +code. They do not run Suricata, establish incident chronology in deployed data, +certify clock synchronization, complete coverage, pass hosted security gates, +authorize protected merge or establish whole-product acceptance. + +## References + +[1] Open Information Security Foundation. *Suricata User Guide: Eve JSON +Format*, current documentation, alert timestamp example. Retrieved through +actual web open on October 4, 2026. +https://docs.suricata.io/en/latest/output/eve/eve-json-format.html + +[2] Klyne, G., and Newman, C. (2002). *Date and Time on the Internet: +Timestamps* (RFC 3339), section 4.2, numeric offsets. RFC Editor. Retrieved +through actual web open on October 4, 2026. +https://www.rfc-editor.org/rfc/rfc3339.html + +The official protocol/producer contracts ground this timestamp-conversion +repair. It introduces no detection model or load-balancing algorithm; unrelated +research PDFs are not attached. diff --git a/src/suricata_eve.rs b/src/suricata_eve.rs index c862032c..6025c1f4 100644 --- a/src/suricata_eve.rs +++ b/src/suricata_eve.rs @@ -118,8 +118,9 @@ pub fn suricata_alert_from_value(value: &serde_json::Value) -> Option Option { let s = raw.trim(); if s.len() < 19 { @@ -155,13 +156,55 @@ pub fn parse_suricata_timestamp(raw: &str) -> Option { { return None; } - days_from_civil(year, month, day).and_then(|days| { - let secs = i64::from(days) * 86_400 - + i64::from(hour) * 3_600 - + i64::from(minute) * 60 - + i64::from(second); - u64::try_from(secs).ok() - }) + let mut remainder = &s[19..]; + if remainder.starts_with('.') { + remainder = &remainder[1..]; + let fraction_len = remainder.bytes().take_while(u8::is_ascii_digit).count(); + if fraction_len == 0 { + return None; + } + remainder = &remainder[fraction_len..]; + } + + let offset_seconds = if remainder.is_empty() || remainder == "Z" { + 0_i64 + } else { + let (sign, offset) = match remainder.as_bytes().first()? { + b'+' => (1_i64, &remainder[1..]), + b'-' => (-1_i64, &remainder[1..]), + _ => return None, + }; + let (hours, minutes) = match offset.len() { + 4 if offset.bytes().all(|byte| byte.is_ascii_digit()) => ( + offset.get(0..2)?.parse::().ok()?, + offset.get(2..4)?.parse::().ok()?, + ), + 5 if offset.as_bytes().get(2) == Some(&b':') + && offset + .as_bytes() + .iter() + .enumerate() + .all(|(index, byte)| index == 2 || byte.is_ascii_digit()) => + { + ( + offset.get(0..2)?.parse::().ok()?, + offset.get(3..5)?.parse::().ok()?, + ) + } + _ => return None, + }; + if hours > 23 || minutes > 59 { + return None; + } + sign * (i64::from(hours) * 3_600 + i64::from(minutes) * 60) + }; + + let days = days_from_civil(year, month, day)?; + let local_seconds = i64::from(days) * 86_400 + + i64::from(hour) * 3_600 + + i64::from(minute) * 60 + + i64::from(second); + u64::try_from(local_seconds.checked_sub(offset_seconds)?).ok() } /// Howard Hinnant civil-from-days inverse: days since Unix epoch for a UTC date. @@ -301,6 +344,58 @@ mod tests { assert_eq!(alert.timestamp_unix, Some(1_718_454_896)); } + #[test] + fn numeric_offsets_preserve_the_utc_instant() { + for timestamp in [ + "2024-06-15T12:34:56Z", + "2024-06-15T13:34:56.123456+0100", + "2024-06-15T21:34:56+09:00", + "2024-06-15T07:04:56-0530", + "2024-06-15T06:49:56.999-05:45", + ] { + assert_eq!( + parse_suricata_timestamp(timestamp), + Some(1_718_454_896), + "{timestamp}" + ); + } + assert_eq!( + parse_suricata_timestamp("1970-01-01T01:00:00+0100"), + Some(0) + ); + assert_eq!( + parse_suricata_timestamp("1969-12-31T23:00:00-0100"), + Some(0) + ); + assert_eq!(parse_suricata_timestamp("1970-01-01T00:00:00+0100"), None); + } + + #[test] + fn malformed_timezone_suffixes_are_not_silently_utc() { + for suffix in [ + "+2400", + "-00:60", + "+090", + "+09:0", + "+aa00", + "Zjunk", + ".Z", + ".abc+0100", + "๐Ÿ’ฅ", + "+0100trailing", + ] { + let timestamp = format!("2024-06-15T12:34:56{suffix}"); + assert_eq!(parse_suricata_timestamp(×tamp), None, "{timestamp}"); + } + // Preserve the existing timezone-less lab contract and zero-offset forms. + for suffix in ["", ".123456", "Z", "+0000", "+00:00", "-0000", "-00:00"] { + assert_eq!( + parse_suricata_timestamp(&format!("2024-06-15T12:34:56{suffix}")), + Some(1_718_454_896) + ); + } + } + #[test] fn parse_suricata_timestamp_handles_common_eve_forms() { assert_eq!( diff --git a/tests/suricata_timestamp_offset.rs b/tests/suricata_timestamp_offset.rs new file mode 100644 index 00000000..1931e928 --- /dev/null +++ b/tests/suricata_timestamp_offset.rs @@ -0,0 +1,103 @@ +//! Suricata numeric offsets must retain the same UTC event occurrence time. +//! Offline synthetic records exercise the public parser and authenticated router. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +async fn read(app: &axum::Router, path: &str, token: &str) -> Value { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(path) + .header("x-admin-token", token) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap() +} + +#[tokio::test] +async fn authenticated_eve_import_retains_offset_occurrence_time() { + let token = format!("time-offset-fixture-{}", std::process::id()); + let app = build_app(AppState::seeded(Some(token.clone()))); + let records = json!([ + {"event_type":"alert", "timestamp":"2024-06-15T13:34:56.123456+0100", "src_ip":"192.0.2.211", "http":{"url":"/offset"}, "alert":{"signature":"positive offset", "severity":1}}, + {"event_type":"alert", "timestamp":"2024-06-15T07:04:56-0530", "src_ip":"192.0.2.212", "http":{"url":"/offset"}, "alert":{"signature":"negative offset", "severity":1}} + ]); + let before = json!({ + "events":read(&app,"/api/events", &token).await, + "dnsbl":read(&app,"/api/dnsbl", &token).await, + "threats":read(&app,"/api/threats", &token).await, + "audit":read(&app,"/api/audit-logs", &token).await + }); + let unauthorized = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/ids/suricata/eve") + .header("content-type", "application/json") + .body(Body::from(records.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(unauthorized.status(), StatusCode::UNAUTHORIZED); + assert_eq!( + before, + json!({ + "events":read(&app,"/api/events", &token).await, + "dnsbl":read(&app,"/api/dnsbl", &token).await, + "threats":read(&app,"/api/threats", &token).await, + "audit":read(&app,"/api/audit-logs", &token).await + }) + ); + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/ids/suricata/eve") + .header("content-type", "application/json") + .header("x-admin-token", &token) + .body(Body::from(records.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let result: Value = + serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap(); + assert_eq!(result["accepted_alerts"], 2); + assert_eq!(result["enforcement_hints"], 6); + let events = read(&app, "/api/events", &token).await; + let events = events.as_array().unwrap(); + assert_eq!(events.len(), 2); + for event in events { + assert_eq!(event["timestamp_unix"], 1_718_454_896_u64); + assert_eq!(event["score"], 80); + assert_eq!(event["action"], "block"); + assert_eq!(event["path"], "/offset"); + } + let dnsbl = read(&app, "/api/dnsbl", &token).await; + let rows: Vec<_> = dnsbl + .as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == "engine:suricata") + .collect(); + assert_eq!(rows.len(), 2); + for row in rows { + assert_eq!(row["ttl_seconds"], 3600); + assert_eq!(row["code"], "127.0.0.2"); + } +} From c2d9c5596da167dbe18a861f9d38801af8edb6c7 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sun, 4 Oct 2026 16:44:16 +0900 Subject: [PATCH 15/22] test(coraza): verify audit policy composition through management API --- tests/coraza_policy_composition.rs | 294 +++++++++++++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 tests/coraza_policy_composition.rs diff --git a/tests/coraza_policy_composition.rs b/tests/coraza_policy_composition.rs new file mode 100644 index 00000000..b11d39d5 --- /dev/null +++ b/tests/coraza_policy_composition.rs @@ -0,0 +1,294 @@ +//! Coraza audit policy composition through authenticated management routes. +//! Synthetic audit records exercise the adapter, not a running Coraza engine. + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +async fn read(app: &axum::Router, path: &str, token: &str) -> Value { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(path) + .header("x-admin-token", token) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap() +} + +async fn import(app: &axum::Router, token: &str, record: Value) -> (StatusCode, Value) { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/waf/coraza/audit") + .header("content-type", "application/json") + .header("x-admin-token", token) + .body(Body::from(record.to_string())) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let value = + serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap(); + (status, value) +} + +async fn indicators(app: &axum::Router, token: &str) -> Value { + json!({ + "dnsbl": read(app, "/api/dnsbl", token).await, + "threats": read(app, "/api/threats", token).await, + }) +} + +async fn block_app(token: &str) -> axum::Router { + let app = build_app(AppState::seeded(Some(token.to_string()))); + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/routes") + .header("content-type", "application/json") + .header("x-admin-token", token) + .body(Body::from( + json!({ + "id": "demo", "path_prefix": "/demo", "upstream": "mock://demo-upstream", + "mode": "block", "enabled": true + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let routes = read(&app, "/api/routes", token).await; + assert_eq!(routes[0]["mode"], "block"); + app +} + +async fn gateway(app: &axum::Router, ip: &str, path: &str) -> (StatusCode, Value) { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(format!("/gateway/demo{path}")) + .header("x-real-ip", ip) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let value = + serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap(); + (status, value) +} + +fn low_severity_record(ip: &str, path: &str, interrupted: bool) -> Value { + json!({ + "transaction": { + "client_ip": ip, + "is_interrupted": interrupted, + "request": {"uri": path}, + }, + "messages": [{"message": "synthetic policy observation", "data": {"severity": 3}}], + }) +} + +#[tokio::test] +async fn mixed_array_publishes_only_block_grade_rows() { + let token = format!("coraza-array-fixture-{}", std::process::id()); + let app = block_app(&token).await; + let (status, result) = import( + &app, + &token, + json!([ + low_severity_record("192.0.2.224", "/coraza-monitor?fixture=1", false), + low_severity_record("192.0.2.225", "/coraza-array?fixture=1", true), + {"transaction": {"response": {"http_code": 200}}} + ]), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["accepted_hits"], 2); + assert_eq!(result["skipped"], 1); + assert_eq!(result["enforcement_hints"], 3); + let events = read(&app, "/api/events", &token).await; + let rows = events.as_array().unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0]["action"], "monitor"); + assert_eq!(rows[1]["action"], "block"); + assert!(rows[0]["id"].as_u64().unwrap() < rows[1]["id"].as_u64().unwrap()); + assert_eq!(result["event_ids"], json!([rows[0]["id"], rows[1]["id"]])); + let hints = indicators(&app, &token).await; + let dnsbl: Vec<_> = hints["dnsbl"] + .as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == "engine:coraza") + .collect(); + assert_eq!(dnsbl.len(), 1); + assert_eq!(dnsbl[0]["address"], "192.0.2.225"); + let threats: Vec<_> = hints["threats"] + .as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == "engine:coraza") + .collect(); + assert_eq!(threats.len(), 2); + assert!( + threats + .iter() + .all(|row| row["value"] == "192.0.2.225" || row["value"] == "/coraza-array") + ); + let before = json!({"events": events, "indicators": hints, + "audit": read(&app, "/api/audit-logs", &token).await}); + let (status, _) = import(&app, &token, json!([])).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + before, + json!({ + "events": read(&app, "/api/events", &token).await, + "indicators": indicators(&app, &token).await, + "audit": read(&app, "/api/audit-logs", &token).await + }) + ); + let (status, response) = gateway(&app, "192.0.2.224", "/coraza-monitor").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(response["score"], 0); + let (status, response) = gateway(&app, "192.0.2.225", "/coraza-array").await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(response["score"], 150); +} + +#[tokio::test] +async fn message_free_interruptions_without_targets_do_not_publish_hints() { + for record in [ + json!({"action": "BLOCK"}), + json!({"action": "DeNy"}), + json!({"action": "DrOp"}), + json!({"transaction": {"response": {"http_code": 406}, "request": {"uri": "/"}}}), + ] { + let token = format!("coraza-action-fixture-{}", std::process::id()); + let app = block_app(&token).await; + let before = indicators(&app, &token).await; + let (status, result) = import(&app, &token, record.clone()).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["accepted_hits"], 1); + assert_eq!(result["enforcement_hints"], 0); + let events = read(&app, "/api/events", &token).await; + assert_eq!(events.as_array().unwrap().len(), 1); + assert_eq!(events[0]["score"], 50); + assert_eq!(events[0]["action"], "block"); + assert_eq!(events[0]["reason"], "coraza/crs: transaction interrupted"); + assert_eq!(events[0]["client_ip"], Value::Null); + assert_eq!( + events[0]["path"], + if record.get("transaction").is_some() { + "/" + } else { + "coraza://transaction" + } + ); + assert_eq!(indicators(&app, &token).await, before); + } +} + +#[tokio::test] +async fn interrupted_low_severity_audit_publishes_medium_hints() { + let token = format!("coraza-interruption-fixture-{}", std::process::id()); + let app = block_app(&token).await; + let (status, result) = import( + &app, + &token, + low_severity_record("192.0.2.222", "/coraza-low?fixture=1", true), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["enforcement_hints"], 3); + let events = read(&app, "/api/events", &token).await; + assert_eq!(events[0]["score"], 25); + assert_eq!(events[0]["action"], "block"); + assert_eq!(events[0]["path"], "/coraza-low?fixture=1"); + let threats = read(&app, "/api/threats", &token).await; + let hints: Vec<_> = threats + .as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == "engine:coraza") + .collect(); + assert_eq!(hints.len(), 2); + for row in &hints { + assert_eq!(row["severity"], "medium"); + assert_eq!(row["ttl_seconds"], 3600); + } + assert!( + hints + .iter() + .any(|row| row["indicator_type"] == "path" && row["value"] == "/coraza-low") + ); + assert!( + hints + .iter() + .any(|row| row["indicator_type"] == "client_ip" && row["value"] == "192.0.2.222") + ); + let dnsbl = read(&app, "/api/dnsbl", &token).await; + let row = dnsbl + .as_array() + .unwrap() + .iter() + .find(|row| row["source"] == "engine:coraza") + .unwrap(); + assert_eq!(row["address"], "192.0.2.222"); + assert_eq!(row["code"], "127.0.0.2"); + assert_eq!(row["ttl_seconds"], 3600); + let (status, response) = gateway(&app, "192.0.2.222", "/coraza-low").await; + assert_eq!(status, StatusCode::FORBIDDEN); + assert_eq!(response["score"], 150); + // A medium path hint alone is below the seeded route's block threshold. + let (status, response) = gateway(&app, "192.0.2.223", "/coraza-low").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(response["score"], 25); +} + +#[tokio::test] +async fn monitor_audit_does_not_publish_enforcement_indicators() { + let token = format!("coraza-policy-fixture-{}", std::process::id()); + let app = block_app(&token).await; + let before = indicators(&app, &token).await; + let (status, result) = import( + &app, + &token, + low_severity_record("192.0.2.221", "/coraza-monitor?fixture=1", false), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["accepted_hits"], 1); + assert_eq!(result["skipped"], 0); + assert_eq!(result["enforcement_hints"], 0); + let events = read(&app, "/api/events", &token).await; + let rows = events.as_array().unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(result["event_ids"], json!([rows[0]["id"]])); + assert_eq!(rows[0]["action"], "monitor"); + assert_eq!(rows[0]["score"], 25); + assert_eq!(rows[0]["client_ip"], "192.0.2.221"); + assert_eq!(rows[0]["path"], "/coraza-monitor?fixture=1"); + assert_eq!(indicators(&app, &token).await, before); + let (status, response) = gateway(&app, "192.0.2.221", "/coraza-monitor").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(response["score"], 0); +} From 9538060500053a762c00ed4d6f8fa13f6152c302 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sun, 4 Oct 2026 18:26:59 +0900 Subject: [PATCH 16/22] test(coraza): verify ingest rejection rollback and retention --- tests/coraza_ingest_atomicity.rs | 340 +++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 tests/coraza_ingest_atomicity.rs diff --git a/tests/coraza_ingest_atomicity.rs b/tests/coraza_ingest_atomicity.rs new file mode 100644 index 00000000..30ae4d66 --- /dev/null +++ b/tests/coraza_ingest_atomicity.rs @@ -0,0 +1,340 @@ +//! Coraza management import must preserve state on rejected requests. +//! All documents, credentials and persisted files belong to this offline fixture. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AdminPrincipal, AppConfig, AppState, build_app}; + +static FIXTURE_ID: AtomicU64 = AtomicU64::new(0); + +struct StateFixture { + root: PathBuf, + path: PathBuf, +} + +impl StateFixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "wardnet-coraza-atomicity-{}-{}", + std::process::id(), + FIXTURE_ID.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&root).expect("create exclusive fixture root"); + Self { + path: root.join("state.json"), + root, + } + } + + fn stored(&self) -> Value { + serde_json::from_slice(&std::fs::read(&self.path).unwrap()).unwrap() + } + + fn children(&self) -> Vec { + let mut paths: Vec<_> = std::fs::read_dir(&self.root) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect(); + paths.sort(); + paths + } +} + +impl Drop for StateFixture { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.root).expect("remove only owned fixture root"); + } +} + +async fn persisted_app(path: &Path, token: &str, event_limit: usize) -> axum::Router { + let state = AppState::load(AppConfig { + admin_token: Some(token.to_string()), + state_path: Some(path.to_path_buf()), + dnsbl_origin: "dnsbl.fixture".to_string(), + event_limit, + }) + .await + .unwrap(); + build_app(state) +} + +async fn read(app: &axum::Router, path: &str, token: &str) -> Value { + let response = app + .clone() + .oneshot( + Request::builder() + .uri(path) + .header("x-admin-token", token) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap() +} + +async fn snapshot(app: &axum::Router, token: &str) -> Value { + json!({ + "events": read(app, "/api/events", token).await, + "threats": read(app, "/api/threats", token).await, + "dnsbl": read(app, "/api/dnsbl", token).await, + "audit": read(app, "/api/audit-logs", token).await, + }) +} + +async fn import(app: &axum::Router, token: Option<&str>, body: Vec) -> (StatusCode, Value) { + let mut request = Request::builder() + .method("POST") + .uri("/api/waf/coraza/audit") + .header("content-type", "application/x-ndjson"); + if let Some(token) = token { + request = request.header("x-admin-token", token); + } + let response = app + .clone() + .oneshot(request.body(Body::from(body)).unwrap()) + .await + .unwrap(); + let status = response.status(); + let value = + serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap(); + (status, value) +} + +fn hit(path: &str, interrupted: bool) -> Value { + json!({ + "transaction": { + "client_ip": "192.0.2.231", "is_interrupted": interrupted, + "request": {"uri": path}, "time_stamp": "2024-06-15T12:34:56Z" + }, + "messages": [{"message": "synthetic atomicity record", "data": {"severity": 3}}] + }) +} + +#[tokio::test] +async fn batch_result_reports_only_retained_event_ids_after_reload() { + let fixture = StateFixture::new(); + let token = format!("coraza-retention-fixture-{}", std::process::id()); + let app = persisted_app(&fixture.path, &token, 2).await; + let before = snapshot(&app, &token).await; + let batch = json!([ + hit("/retention-one", false), + hit("/retention-two", false), + hit("/retention-three", false) + ]); + let (status, result) = import(&app, Some(&token), batch.to_string().into_bytes()).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["accepted_hits"], 3); + assert_eq!(result["skipped"], 0); + assert_eq!(result["event_ids"], json!([2, 3])); + assert_eq!(result["enforcement_hints"], 0); + let committed = snapshot(&app, &token).await; + assert_eq!(committed["dnsbl"], before["dnsbl"]); + assert_eq!(committed["threats"], before["threats"]); + let events = committed["events"].as_array().unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0]["id"], 2); + assert_eq!(events[0]["path"], "/retention-two"); + assert_eq!(events[1]["id"], 3); + assert_eq!(events[1]["path"], "/retention-three"); + for event in events { + assert_eq!(event["action"], "monitor"); + assert_eq!(event["score"], 25); + assert_eq!(event["timestamp_unix"], 1_718_454_896_u64); + } + let audit = committed["audit"].as_array().unwrap(); + assert_eq!(audit.len(), 1); + assert_eq!(audit[0]["id"], 1); + assert_eq!(audit[0]["action"], "import_coraza_audit"); + assert_eq!(audit[0]["resource"], "waf_coraza"); + assert_eq!(audit[0]["resource_id"], "3_hits"); + assert_eq!(audit[0]["outcome"], "success"); + let persisted = fixture.stored(); + assert_eq!(persisted["events"], committed["events"]); + assert_eq!(persisted["audit_logs"], committed["audit"]); + assert_eq!(persisted["next_event_id"], 4); + assert_eq!(persisted["next_audit_log_id"], 2); + assert_eq!(fixture.children(), vec![fixture.path.clone()]); + drop(app); + let reloaded = persisted_app(&fixture.path, &token, 2).await; + assert_eq!(snapshot(&reloaded, &token).await, committed); + assert_eq!(fixture.stored(), persisted); + let (status, next) = import( + &reloaded, + Some(&token), + hit("/retention-four", false).to_string().into_bytes(), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(next["event_ids"], json!([4])); + assert_eq!(fixture.stored()["next_event_id"], 5); + assert_eq!(fixture.stored()["next_audit_log_id"], 3); +} + +#[tokio::test] +async fn persistence_failure_rolls_back_coraza_batch_before_retry() { + let fixture = StateFixture::new(); + let token = format!("coraza-rollback-fixture-{}", std::process::id()); + let app = persisted_app(&fixture.path, &token, 10).await; + let before = snapshot(&app, &token).await; + let baseline = std::fs::read(&fixture.path).unwrap(); + // Replace only the owned destination after startup so atomic rename fails. + std::fs::remove_file(&fixture.path).unwrap(); + std::fs::create_dir(&fixture.path).unwrap(); + let batch = json!([hit("/rollback-one", true), hit("/rollback-two", true)]); + let (status, response) = import(&app, Some(&token), batch.to_string().into_bytes()).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert!( + response["error"] + .as_str() + .unwrap() + .contains("failed to replace state file") + ); + assert_eq!(snapshot(&app, &token).await, before); + assert!(fixture.path.is_dir()); + assert_eq!(fixture.children(), vec![fixture.path.clone()]); + assert_eq!(std::fs::read_dir(&fixture.path).unwrap().count(), 0); + + // Recovery is fixture-only: restore its baseline file, then use the same app. + // The subsequent IDs verify in-memory counters, not just unchanged disk. + std::fs::remove_dir(&fixture.path).unwrap(); + std::fs::write(&fixture.path, baseline).unwrap(); + let (status, result) = import(&app, Some(&token), batch.to_string().into_bytes()).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["accepted_hits"], 2); + assert_eq!(result["event_ids"], json!([1, 2])); + assert_eq!(result["enforcement_hints"], 6); + let committed = snapshot(&app, &token).await; + let persisted = fixture.stored(); + assert_eq!(persisted["events"], committed["events"]); + assert_eq!(persisted["threats"], committed["threats"]); + assert_eq!(persisted["dnsbl"], committed["dnsbl"]); + assert_eq!(persisted["audit_logs"], committed["audit"]); + assert_eq!(persisted["next_event_id"], 3); + assert_eq!(persisted["next_audit_log_id"], 2); + assert_eq!(persisted["audit_logs"].as_array().unwrap().len(), 1); + assert_eq!(persisted["audit_logs"][0]["id"], 1); + assert_eq!(persisted["audit_logs"][0]["action"], "import_coraza_audit"); + assert_eq!(persisted["audit_logs"][0]["outcome"], "success"); + assert_eq!(fixture.children(), vec![fixture.path.clone()]); + drop(app); + let reloaded = persisted_app(&fixture.path, &token, 10).await; + assert_eq!(snapshot(&reloaded, &token).await, committed); + assert_eq!(fixture.stored(), persisted); +} + +#[tokio::test] +async fn invalid_coraza_bodies_do_not_commit_a_partial_batch() { + let malformed_batch = format!("{}\n{{", hit("/partial-batch", true)); + for (body, diagnostic) in [ + (vec![0xff], "Coraza audit body must be UTF-8 text"), + (b" \n".to_vec(), "empty Coraza audit body"), + (b"{}".to_vec(), "no Coraza WAF hits found"), + (b"[]".to_vec(), "no Coraza WAF hits found"), + ( + malformed_batch.into_bytes(), + "invalid Coraza audit JSON on line 2", + ), + ] { + let fixture = StateFixture::new(); + let token = format!("coraza-invalid-fixture-{}", std::process::id()); + let app = persisted_app(&fixture.path, &token, 10).await; + let before = snapshot(&app, &token).await; + let stored = std::fs::read(&fixture.path).unwrap(); + let (status, response) = import(&app, Some(&token), body).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(response["error"].as_str().unwrap().contains(diagnostic)); + assert_eq!(snapshot(&app, &token).await, before); + assert_eq!(std::fs::read(&fixture.path).unwrap(), stored); + assert_eq!(fixture.children(), vec![fixture.path.clone()]); + // A successful import after rejection proves neither ID counter advanced. + let (status, result) = import( + &app, + Some(&token), + hit("/after-rejection", true).to_string().into_bytes(), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["event_ids"], json!([1])); + assert_eq!(result["enforcement_hints"], 3); + let persisted = fixture.stored(); + assert_eq!(persisted["next_event_id"], 2); + assert_eq!(persisted["next_audit_log_id"], 2); + assert_eq!(persisted["audit_logs"][0]["id"], 1); + } +} + +#[tokio::test] +async fn write_authorization_precedes_coraza_body_validation() { + let fixture = StateFixture::new(); + let writer = format!("coraza-write-fixture-{}", std::process::id()); + let reader = format!("coraza-read-fixture-{}", std::process::id()); + let state = AppState::load(AppConfig { + admin_token: None, + state_path: Some(fixture.path.clone()), + dnsbl_origin: "dnsbl.fixture".to_string(), + event_limit: 10, + }) + .await + .unwrap() + .with_admin_tokens(HashMap::from([ + ( + writer.clone(), + AdminPrincipal { + actor: "fixture-writer".into(), + can_write: true, + }, + ), + ( + reader.clone(), + AdminPrincipal { + actor: "fixture-reader".into(), + can_write: false, + }, + ), + ])); + let app = build_app(state); + let before = snapshot(&app, &writer).await; + let stored = std::fs::read(&fixture.path).unwrap(); + for (token, expected) in [ + (None, StatusCode::UNAUTHORIZED), + (Some("unknown-fixture-principal"), StatusCode::UNAUTHORIZED), + (Some(reader.as_str()), StatusCode::FORBIDDEN), + ] { + let (status, response) = import(&app, token, vec![0xff]).await; + assert_eq!(status, expected); + assert!( + response["error"] + .as_str() + .unwrap() + .contains("X-Admin-Token") + ); + let text = response.to_string(); + assert!(!text.contains(&writer) && !text.contains(&reader)); + assert_eq!(snapshot(&app, &writer).await, before); + assert_eq!(std::fs::read(&fixture.path).unwrap(), stored); + } + let (status, response) = import( + &app, + Some(&writer), + hit("/auth-positive", false).to_string().into_bytes(), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(response["event_ids"], json!([1])); + let persisted = fixture.stored(); + assert_eq!(persisted["next_event_id"], 2); + assert_eq!(persisted["next_audit_log_id"], 2); + assert_eq!(persisted["audit_logs"][0]["actor"], "fixture-writer"); +} From 33da961ba4447795b10877af4daafbbd216c1537 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Mon, 5 Oct 2026 01:27:26 +0900 Subject: [PATCH 17/22] test(ci): bind checkout credential guard to each step --- tests/workflow_runner_contract.rs | 238 +++++++++++++++++++++++++++++- 1 file changed, 234 insertions(+), 4 deletions(-) diff --git a/tests/workflow_runner_contract.rs b/tests/workflow_runner_contract.rs index 7fee9a57..6f71c4cd 100644 --- a/tests/workflow_runner_contract.rs +++ b/tests/workflow_runner_contract.rs @@ -62,16 +62,246 @@ fn every_local_workflow_requires_the_isolated_self_hosted_pool() { ); } +// A deliberately limited block-mapping reader for these repository workflows. +// Unsupported checkout options fail closed; this is not a general YAML parser. +fn checkout_credentials_are_disabled(workflow: &str) -> bool { + fn scalar(value: &str) -> Option<&str> { + let mut quote = None; + let mut end = value.len(); + for (index, character) in value.char_indices() { + match (quote, character) { + (None, '\'' | '"') => quote = Some(character), + (Some(delimiter), closing) if delimiter == closing => quote = None, + (None, '#') if index == 0 || value[..index].ends_with(char::is_whitespace) => { + end = index; + break; + } + _ => {} + } + } + if quote.is_some() { + return None; + } + let value = value[..end].trim(); + if value.starts_with(['\'', '"']) { + let delimiter = value.chars().next()?; + value.strip_prefix(delimiter)?.strip_suffix(delimiter) + } else { + Some(value) + } + } + + fn checkout_step(lines: &[&str], sequence_indent: usize) -> Option { + let field_indent = sequence_indent + 2; + let mut uses = Vec::new(); + let mut options = Vec::new(); + for (index, line) in lines.iter().enumerate() { + let indent = line.len() - line.trim_start().len(); + let content = if index == 0 { + line.trim_start().strip_prefix("- ")? + } else { + if indent != field_indent { + continue; + } + line.trim_start() + }; + if let Some(value) = content.strip_prefix("uses:") { + uses.push(scalar(value)?); + } + if let Some(value) = content.strip_prefix("with:") { + options.push((index, scalar(value)?)); + } + } + if !uses + .iter() + .any(|action| action.starts_with("actions/checkout@")) + { + return Some(false); + } + if uses.len() != 1 || options.len() != 1 || !options[0].1.is_empty() { + return None; + } + let mut credentials = Vec::new(); + for line in &lines[options[0].0 + 1..] { + if line.trim().is_empty() || line.trim_start().starts_with('#') { + continue; + } + let indent = line.len() - line.trim_start().len(); + if indent <= field_indent { + break; + } + if indent == field_indent + 2 + && let Some(value) = line.trim_start().strip_prefix("persist-credentials:") + { + // Require the literal YAML boolean, not a quoted string/expression. + if value.trim_start().starts_with(['\'', '"']) { + return None; + } + credentials.push(scalar(value)?); + } + } + (credentials == ["false"]).then_some(true) + } + + if workflow.lines().any(|line| line.starts_with('\t')) { + return false; + } + let lines: Vec<_> = workflow.lines().collect(); + let mut checkouts = 0; + let mut index = 0; + while index < lines.len() { + let line = lines[index]; + let indent = line.len() - line.trim_start().len(); + let Some(value) = line.trim_start().strip_prefix("steps:") else { + index += 1; + continue; + }; + if indent != 4 || scalar(value) != Some("") { + return false; + } + let steps_indent = indent; + index += 1; + while index < lines.len() { + let line = lines[index]; + if line.trim().is_empty() || line.trim_start().starts_with('#') { + index += 1; + continue; + } + let indent = line.len() - line.trim_start().len(); + if indent <= steps_indent { + break; + } + if indent != steps_indent + 2 || !line.trim_start().starts_with("- ") { + return false; + } + let start = index; + index += 1; + while index < lines.len() { + let next = lines[index]; + if !next.trim().is_empty() + && !next.trim_start().starts_with('#') + && next.len() - next.trim_start().len() <= indent + { + break; + } + index += 1; + } + match checkout_step(&lines[start..index], indent) { + Some(true) => checkouts += 1, + Some(false) => {} + None => return false, + } + } + } + checkouts > 0 +} + #[test] fn checkout_credentials_are_not_persisted_on_self_hosted_workers() { let repository = Path::new(env!("CARGO_MANIFEST_DIR")); for relative in RUNNER_BACKED_WORKFLOWS { let workflow = fs::read_to_string(repository.join(relative)).unwrap(); - assert_eq!(workflow.matches("actions/checkout@").count(), 1); - assert_eq!( - workflow.matches("persist-credentials: false").count(), - 1, + assert!( + checkout_credentials_are_disabled(&workflow), "{relative} must not leave checkout credentials in the workspace" ); } } + +#[test] +fn commented_credential_option_cannot_authorize_checkout() { + let safe = "jobs:\n rust:\n steps:\n - uses: actions/checkout@fixture\n with:\n persist-credentials: false\n"; + let unsafe_workflow = + safe.replace("persist-credentials: false", "# persist-credentials: false"); + assert!(checkout_credentials_are_disabled(safe)); + assert!( + !checkout_credentials_are_disabled(&unsafe_workflow), + "a comment must not authorize credential persistence" + ); +} + +fn checkout_fixture(action: &str, option: &str) -> String { + format!( + "jobs:\n rust:\n steps:\n - uses: {action}\n with:\n persist-credentials: {option}\n" + ) +} + +#[test] +fn every_checkout_step_requires_its_own_literal_false() { + for action in [ + "actions/checkout@fixture", + "'actions/checkout@fixture'", + "\"actions/checkout@fixture\"", + ] { + let safe = checkout_fixture(action, "false"); + assert!(checkout_credentials_are_disabled(&safe)); + for option in ["true", "'false'", "\"false\"", "${{ false }}", ""] { + let unsafe_step = format!( + " - uses: {action}\n with:\n persist-credentials: {option}\n" + ); + assert!( + !checkout_credentials_are_disabled(&(safe.clone() + &unsafe_step)), + "unsafe later checkout option {option:?} must be rejected" + ); + } + let second_safe = format!( + " - uses:\t{action}\n with:\n persist-credentials: false # literal control\n" + ); + assert!(checkout_credentials_are_disabled(&(safe + &second_safe))); + } +} + +#[test] +fn unrelated_step_or_job_option_cannot_authorize_checkout() { + let unsafe_workflow = "jobs:\n rust:\n steps:\n - uses: actions/checkout@fixture\n - name: unrelated\n with:\n persist-credentials: false\n"; + assert!(!checkout_credentials_are_disabled(unsafe_workflow)); + let other_job = " other:\n steps:\n - name: not checkout\n with:\n persist-credentials: false\n"; + let missing_option = "jobs:\n rust:\n steps:\n - uses: actions/checkout@fixture\n"; + assert!(!checkout_credentials_are_disabled( + &(missing_option.to_owned() + other_job) + )); + let no_checkout = "jobs:\n rust:\n steps:\n - run: |\n echo actions/checkout@fixture\n echo persist-credentials: false\n"; + assert!(!checkout_credentials_are_disabled(no_checkout)); +} + +#[test] +fn commented_steps_header_cannot_hide_an_unsafe_later_job() { + let safe = checkout_fixture("actions/checkout@fixture", "false"); + let unsafe_job = " later:\n steps: # ordinary inline comment\n - uses: actions/checkout@fixture\n with:\n persist-credentials: true\n"; + assert!(checkout_credentials_are_disabled(&safe)); + assert!( + !checkout_credentials_are_disabled(&(safe.clone() + unsafe_job)), + "a commented steps header must not hide a later unsafe checkout" + ); + let safe_job = unsafe_job.replace("persist-credentials: true", "persist-credentials: false"); + assert!(checkout_credentials_are_disabled(&(safe + &safe_job))); +} + +#[test] +fn ambiguous_or_nonliteral_checkout_configuration_fails_closed() { + let safe = checkout_fixture("actions/checkout@fixture", "false"); + for option in ["false#not-a-comment", "false extra", "False", "null"] { + assert!( + !checkout_credentials_are_disabled(&checkout_fixture( + "actions/checkout@fixture", + option + )), + "unsupported credential scalar {option:?} must not pass" + ); + } + let duplicate = safe.replace( + "persist-credentials: false", + "persist-credentials: false\n persist-credentials: true", + ); + assert!(!checkout_credentials_are_disabled(&duplicate)); + let inline = safe.replace( + "with:\n persist-credentials: false", + "with: { persist-credentials: false }", + ); + assert!(!checkout_credentials_are_disabled(&inline)); + let wrong_depth = safe.replace( + " persist-credentials", + " persist-credentials", + ); + assert!(!checkout_credentials_are_disabled(&wrong_depth)); +} From 28cfd96136b8423601f1ae527f204ac030a42780 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Mon, 5 Oct 2026 03:19:21 +0900 Subject: [PATCH 18/22] fix(opencti): validate known digest syntax before feed import --- .../opencti-known-digest-boundary.md | 81 ++++ src/opencti_import.rs | 22 + tests/opencti_hash_digest_boundary.rs | 379 ++++++++++++++++++ 3 files changed, 482 insertions(+) create mode 100644 docs/doctoring/opencti-known-digest-boundary.md create mode 100644 tests/opencti_hash_digest_boundary.rs diff --git a/docs/doctoring/opencti-known-digest-boundary.md b/docs/doctoring/opencti-known-digest-boundary.md new file mode 100644 index 00000000..813d8d00 --- /dev/null +++ b/docs/doctoring/opencti-known-digest-boundary.md @@ -0,0 +1,81 @@ +# OpenCTI known-digest admission boundary + +## Observed defect and root cause + +The authenticated export-import route accepted an `MD5` observable whose value +was `status`. The actual repository regression observed HTTP201 where HTTP400 +was required. A valid 32-character hexadecimal MD5 positive returned HTTP201 +and retained lowercase value, source, severity and TTL across reload. + +The direct digest branch and explicit file-hash array/map branches checked +nonempty strings but not algorithm-specific digest syntax. The existing generic +threat validator does not impose that syntax either. Generic request scoring +uses non-IP indicators as text. This repair prevents malformed known digests +from entering that consumer; it does not change scoring policy or compute file +hashes. No unauthenticated attack, live upstream corruption or deployed impact +was observed. + +## Minimal boundary repair + +`src/opencti_import.rs` shares one private predicate across direct MD5/SHA1/ +SHA256/SHA512 and explicit file hash arrays/maps. After existing value trimming, +known algorithms require ASCII hexadecimal characters and exact textual lengths: +MD5 32; SHA1/SHA-1 40; SHA256/SHA-256 64; SHA512/SHA-512 128. These lengths follow +the specified digest bit widths and four bits per hexadecimal digit.[1][2] + +Malformed known pairs follow the existing skipped-object path. An explicit +invalid array or map never substitutes the display hash. If nothing maps, the +existing no-mappable response is HTTP400 before feed upsert. Mixed input retains +valid rows and the existing object-level skip count, not a new per-pair count. +Unknown algorithm strings retain the existing nonempty/lowercase compatibility +policy. Their validity is not certified by this predicate. Direct hyphenated +entity recognition, unsupported hash shapes, STIX patterns/bundles, MISP, generic +threat validation, feed ownership, TTL expiry and request scoring are unchanged. + +This is a deliberate compatibility restriction for malformed known digests. +An operator's arbitrary keyword is not an MD5 digest. It must not be represented +as a known digest just because a nonempty string previously passed ingestion. +The repair does not endorse MD5 or SHA-1 for collision-resistant security use. + +## Actual regression boundaries + +Three separate intended HTTP201-versus400 RED receipts are retained outside the +repository: direct MD5; explicit array; explicit map after the array was fixed. +Each repair was followed by the same real Axum control. The direct case pairs +invalid rejection with valid-digest acceptance and an explicit block route. + +Six integration tests additionally cover known lengths, short/long/nonhex/ +UTF-8/internal-space denials, hyphenated algorithm spelling, uppercase and outer +whitespace normalization, unknown-algorithm compatibility, mixed valid evidence, +unauthenticated denial, full management snapshot and persisted-byte nonmutation, +normal gateway admission, and fresh application reload. Fixtures allocate +exclusive owned directories and remove only those directories. + +These are synthetic documents through actual parser, management API, persistence +and gateway code. They are not live OpenCTI pull, deployed authentication, file +malware detection, complete STIX conformance, atomic concurrent-read guarantees, +coverage100%, hosted security, counted approval, protected merge or release. +The older GraphQL hash-array compatibility document describes its historical +repair; the present digest restriction supersedes only its then-excluded digest +validation limitation, without retrospectively changing that repair's evidence. + +## Primary references + +[1] R. Rivest. The MD5 Message-Digest Algorithm. RFC1321, April1992, +section1 (128-bit output) and appendixA.4 (`MDPrint` hexadecimal representation). +https://www.rfc-editor.org/rfc/rfc1321 + +[2] National Institute of Standards and Technology. Secure Hash Standard (SHS). +FIPS PUB180-4, August2015, section1 and algorithm summary table/sections6.1โ€“6.4. +https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.180-4.pdf + +RFC1321 section1/MDPrint and FIPS180-4 Figure1 support the widths and +representation cited above. Reference verification occurred after implementation +and the initial freeze; earlier draft retrieval claims are excluded. This +reference check does not attest prior research or upstream OpenCTI enforcement. +This is standards-grounded input validation, not a new detection engine. +Sources are cited and summarized; redistribution permission is not asserted. + +Sources: +[1] https://www.rfc-editor.org/rfc/rfc1321 โ€” Rivest: The MD5 Message-Digest Algorithm, RFC1321, April1992 +[2] https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.180-4.pdf โ€” NIST: Secure Hash Standard, FIPS PUB180-4, August2015 diff --git a/src/opencti_import.rs b/src/opencti_import.rs index 2012f4f1..1b9bee21 100644 --- a/src/opencti_import.rs +++ b/src/opencti_import.rs @@ -360,6 +360,9 @@ fn materialize_node(node: &serde_json::Value, source: &str, ttl_seconds: u64) -> .map(str::trim) .filter(|s| !s.is_empty()); if let (Some(algorithm), Some(hash)) = (algorithm, hash) { + if !valid_known_digest(algorithm, hash) { + continue; + } threats.push(ThreatIndicator { value: hash.to_ascii_lowercase(), indicator_type: algorithm.to_ascii_lowercase(), @@ -376,6 +379,9 @@ fn materialize_node(node: &serde_json::Value, source: &str, ttl_seconds: u64) -> let mut any = false; for (algo, hash_val) in hashes { if let Some(hash) = hash_val.as_str().map(str::trim).filter(|s| !s.is_empty()) { + if !valid_known_digest(algo, hash) { + continue; + } threats.push(ThreatIndicator { value: hash.to_ascii_lowercase(), indicator_type: algo.to_ascii_lowercase(), @@ -402,6 +408,9 @@ fn materialize_node(node: &serde_json::Value, source: &str, ttl_seconds: u64) -> } } "md5" | "sha1" | "sha256" | "sha512" => { + if !valid_known_digest(&normalized_type, value) { + return NodeOutcome::Skipped; + } threats.push(ThreatIndicator { value: value.to_ascii_lowercase(), indicator_type: normalized_type, @@ -476,6 +485,19 @@ fn is_plausible_domain(host: &str) -> bool { .all(|byte| byte.is_ascii_alphanumeric() || byte == b'.' || byte == b'-') } +// Only known fixed-length hexadecimal digests gain stricter admission. +// Unknown algorithms retain the existing nonempty export-compatibility policy. +fn valid_known_digest(algorithm: &str, value: &str) -> bool { + let length = match algorithm.trim().to_ascii_lowercase().as_str() { + "md5" => 32, + "sha1" | "sha-1" => 40, + "sha256" | "sha-256" => 64, + "sha512" | "sha-512" => 128, + _ => return true, + }; + value.len() == length && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + fn looks_like_hash(value: &str) -> bool { let len = value.len(); matches!(len, 32 | 40 | 64 | 128) && value.bytes().all(|b| b.is_ascii_hexdigit()) diff --git a/tests/opencti_hash_digest_boundary.rs b/tests/opencti_hash_digest_boundary.rs new file mode 100644 index 00000000..5480262a --- /dev/null +++ b/tests/opencti_hash_digest_boundary.rs @@ -0,0 +1,379 @@ +//! Synthetic export documents exercise actual management, storage and gateway code. +//! No live OpenCTI service, operator credentials or file malware scanning. +use std::{ + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; + +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppConfig, AppState, build_app}; + +static FIXTURE_ID: AtomicU64 = AtomicU64::new(0); + +struct StateFixture { + root: PathBuf, + path: PathBuf, +} +impl StateFixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "wardnet-opencti-digest-{}-{}", + std::process::id(), + FIXTURE_ID.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&root).expect("create exclusive owned fixture"); + Self { + path: root.join("state.json"), + root, + } + } + fn bytes(&self) -> Vec { + std::fs::read(&self.path).unwrap() + } +} +impl Drop for StateFixture { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.root).expect("remove only owned fixture"); + } +} + +async fn request( + app: &axum::Router, + method: &str, + path: &str, + token: Option<&str>, + document: Option<&Value>, +) -> (StatusCode, Value) { + let mut builder = Request::builder().method(method).uri(path); + if let Some(token) = token { + builder = builder.header("x-admin-token", token); + } + let body = match document { + Some(value) => { + builder = builder.header("content-type", "application/json"); + Body::from(value.to_string()) + } + None => Body::empty(), + }; + let response = app + .clone() + .oneshot(builder.body(body).unwrap()) + .await + .unwrap(); + let status = response.status(); + let body = + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap(); + (status, body) +} + +async fn read(app: &axum::Router, path: &str, token: &str) -> Value { + let (status, body) = request(app, "GET", path, Some(token), None).await; + assert_eq!(status, StatusCode::OK); + body +} + +async fn app(fixture: &StateFixture, token: &str) -> axum::Router { + build_app( + AppState::load(AppConfig { + admin_token: Some(token.to_string()), + state_path: Some(fixture.path.clone()), + dnsbl_origin: "dnsbl.fixture".to_string(), + event_limit: 100, + }) + .await + .unwrap(), + ) +} + +async fn block_app(fixture: &StateFixture, token: &str) -> axum::Router { + let app = app(fixture, token).await; + let route = json!({"id": "digest", "path_prefix": "/digest", "upstream": "mock://digest", "mode": "block", "enabled": true}); + let (status, _) = request(&app, "POST", "/api/routes", Some(token), Some(&route)).await; + assert_eq!(status, StatusCode::CREATED); + let routes = read(&app, "/api/routes", token).await; + let route = routes + .as_array() + .unwrap() + .iter() + .find(|r| r["id"] == "digest") + .unwrap(); + assert_eq!(route["mode"], "block"); + app +} + +async fn snapshot(app: &axum::Router, token: &str) -> Value { + json!({"threats": read(app, "/api/threats", token).await, + "dnsbl": read(app, "/api/dnsbl", token).await, + "feeds": read(app, "/api/threat-feeds", token).await, + "audit": read(app, "/api/audit-logs", token).await, + "events": read(app, "/api/events", token).await}) +} + +async fn import(app: &axum::Router, document: &Value, token: Option<&str>) -> (StatusCode, Value) { + request( + app, + "POST", + "/api/threat-intel/opencti?feed_id=digest-boundary&source=fixture:digest&ttl_seconds=600", + token, + Some(document), + ) + .await +} + +async fn allowed_status(app: &axum::Router) { + let (status, body) = request(app, "GET", "/gateway/digest/status", None, None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["score"], 0); +} + +#[tokio::test] +async fn invalid_direct_md5_is_rejected_without_mutating_persisted_enforcement() { + let fixture = StateFixture::new(); + let token = format!("opencti-digest-fixture-{}", std::process::id()); + let app = block_app(&fixture, &token).await; + allowed_status(&app).await; + let before = snapshot(&app, &token).await; + let bytes = fixture.bytes(); + let document = json!({"entities": [{"entity_type": "MD5", "observable_value": "status", "x_opencti_score": 80}]}); + let (status, _) = import(&app, &document, Some(&token)).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "known MD5 must reject an arbitrary keyword" + ); + assert_eq!(snapshot(&app, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + allowed_status(&app).await; + let reloaded = app_after_reload(&fixture, &token).await; + assert_eq!( + read(&reloaded, "/api/threats", &token).await, + before["threats"] + ); + assert_eq!( + read(&reloaded, "/api/threat-feeds", &token).await, + before["feeds"] + ); + allowed_status(&reloaded).await; +} + +#[tokio::test] +async fn explicit_file_hashes_reject_invalid_known_digests_without_display_fallback() { + for hashes in [ + json!([{"algorithm": "MD5", "hash": "status"}]), + json!({"MD5": "status"}), + ] { + let fixture = StateFixture::new(); + let token = format!("opencti-digest-fixture-{}", std::process::id()); + let app = block_app(&fixture, &token).await; + allowed_status(&app).await; + let before = snapshot(&app, &token).await; + let bytes = fixture.bytes(); + let document = json!({"data": {"stixCyberObservables": {"edges": [{"node": { + "entity_type": "StixFile", "observable_value": "a".repeat(32), + "hashes": hashes, "x_opencti_score": 80 + }}]}}}); + let (status, _) = import(&app, &document, Some(&token)).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "explicit invalid digest must not use display hash" + ); + assert_eq!(snapshot(&app, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + allowed_status(&app).await; + } +} + +fn hash_document(kind: &str, algorithm: &str, value: &str) -> Value { + match kind { + "direct" => { + json!({"entity_type": algorithm, "observable_value": value, "x_opencti_score": 80}) + } + "array" => json!({"entity_type": "StixFile", "observable_value": "a".repeat(32), + "hashes": [{"algorithm": algorithm, "hash": value}], "x_opencti_score": 80}), + "map" => json!({"entity_type": "Artifact", "observable_value": "a".repeat(32), + "hashes": {algorithm: value}, "x_opencti_score": 80}), + _ => panic!("unsupported fixture shape"), + } +} + +#[tokio::test] +async fn known_digest_length_and_ascii_hex_boundaries_apply_to_every_mapping() { + let fixture = StateFixture::new(); + let token = format!("opencti-digest-fixture-{}", std::process::id()); + let app = block_app(&fixture, &token).await; + for kind in ["direct", "array", "map"] { + for (algorithm, length) in [("MD5", 32), ("SHA1", 40), ("SHA256", 64), ("SHA512", 128)] { + let valid = "AB".repeat(length / 2); + let (status, result) = import( + &app, + &hash_document(kind, algorithm, &format!(" {valid} ")), + Some(&token), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "{kind}/{algorithm}"); + assert_eq!(result["upserted_threats"], 1); + let threats = read(&app, "/api/threats", &token).await; + let row = threats + .as_array() + .unwrap() + .iter() + .find(|r| r["source"] == "fixture:digest") + .unwrap(); + assert_eq!(row["value"], valid.to_ascii_lowercase()); + assert_eq!(row["indicator_type"], algorithm.to_ascii_lowercase()); + let before = snapshot(&app, &token).await; + let bytes = fixture.bytes(); + for invalid in [ + "a".repeat(length - 1), + "a".repeat(length + 1), + "g".repeat(length), + "รฉ".repeat(length / 2), + format!("{} {}", "a".repeat(length / 2), "a".repeat(length / 2 - 1)), + ] { + let (status, _) = import( + &app, + &hash_document(kind, algorithm, &invalid), + Some(&token), + ) + .await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "{kind}/{algorithm} invalid digest admitted" + ); + assert_eq!(snapshot(&app, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + } + } + } +} + +#[tokio::test] +async fn hyphenated_algorithms_retain_spelling_and_unknown_algorithms_remain_compatible() { + let fixture = StateFixture::new(); + let token = format!("opencti-digest-fixture-{}", std::process::id()); + let app = app(&fixture, &token).await; + for kind in ["array", "map"] { + for (algorithm, length) in [("SHA-1", 40), ("SHA-256", 64), ("SHA-512", 128)] { + let valid = "CD".repeat(length / 2); + let (status, _) = + import(&app, &hash_document(kind, algorithm, &valid), Some(&token)).await; + assert_eq!(status, StatusCode::CREATED); + let before = snapshot(&app, &token).await; + let bytes = fixture.bytes(); + let threats = before["threats"].as_array().unwrap(); + let row = threats + .iter() + .find(|r| r["source"] == "fixture:digest") + .unwrap(); + assert_eq!(row["indicator_type"], algorithm.to_ascii_lowercase()); + assert_eq!(row["value"], valid.to_ascii_lowercase()); + let (status, _) = import( + &app, + &hash_document(kind, algorithm, "status"), + Some(&token), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(snapshot(&app, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + } + let (status, _) = import( + &app, + &hash_document(kind, "SSDEEP", "3:AB:CD"), + Some(&token), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + let threats = read(&app, "/api/threats", &token).await; + let row = threats + .as_array() + .unwrap() + .iter() + .find(|r| r["source"] == "fixture:digest") + .unwrap(); + assert_eq!(row["indicator_type"], "ssdeep"); + assert_eq!(row["value"], "3:ab:cd"); + } +} + +#[tokio::test] +async fn mixed_hash_rows_keep_valid_evidence_and_authorization_precedes_validation() { + let fixture = StateFixture::new(); + let token = format!("opencti-digest-fixture-{}", std::process::id()); + let app = app(&fixture, &token).await; + let valid = "AB".repeat(32); + for hashes in [ + json!([{"algorithm": "MD5", "hash": "status"}, {"algorithm": "SHA-256", "hash": valid}]), + json!({"MD5": "status", "SHA-256": valid}), + ] { + let document = json!({"entities": [ + {"entity_type": "MD5", "observable_value": "status"}, + {"entity_type": "StixFile", "observable_value": "file.bin", "hashes": hashes, "x_opencti_score": 80}]}); + let before = snapshot(&app, &token).await; + let bytes = fixture.bytes(); + let (status, _) = import(&app, &document, None).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(snapshot(&app, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + let (status, result) = import(&app, &document, Some(&token)).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["upserted_threats"], 1); + assert_eq!(result["upserted_dnsbl"], 0); + assert_eq!(result["skipped_objects"], 1); + let after = snapshot(&app, &token).await; + assert_eq!(after["dnsbl"], before["dnsbl"]); + let rows: Vec<_> = after["threats"] + .as_array() + .unwrap() + .iter() + .filter(|r| r["source"] == "fixture:digest") + .collect(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["indicator_type"], "sha-256"); + assert_eq!(rows[0]["value"], valid.to_ascii_lowercase()); + let reload = app_after_reload(&fixture, &token).await; + assert_eq!( + read(&reload, "/api/threats", &token).await, + after["threats"] + ); + } +} + +async fn app_after_reload(fixture: &StateFixture, token: &str) -> axum::Router { + app(fixture, token).await +} + +#[tokio::test] +async fn valid_direct_md5_preserves_digest_metadata_and_reload() { + let fixture = StateFixture::new(); + let token = format!("opencti-digest-fixture-{}", std::process::id()); + let app = block_app(&fixture, &token).await; + let value = "AB".repeat(16); + let document = json!({"entities": [{"entity_type": "MD5", "observable_value": value, "x_opencti_score": 80}]}); + let (status, result) = import(&app, &document, Some(&token)).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["upserted_threats"], 1); + assert_eq!(result["upserted_dnsbl"], 0); + let threats = read(&app, "/api/threats", &token).await; + let rows: Vec<_> = threats + .as_array() + .unwrap() + .iter() + .filter(|r| r["source"] == "fixture:digest") + .collect(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["value"], value.to_ascii_lowercase()); + assert_eq!(rows[0]["indicator_type"], "md5"); + assert_eq!(rows[0]["severity"], "critical"); + assert_eq!(rows[0]["ttl_seconds"], 600); + let reloaded = app_after_reload(&fixture, &token).await; + assert_eq!(read(&reloaded, "/api/threats", &token).await, threats); + allowed_status(&reloaded).await; +} From 7e00741ad1c0e702d8b6b25fed637bec51a23007 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Mon, 5 Oct 2026 06:12:21 +0900 Subject: [PATCH 19/22] fix(stix): preserve comparison semantics and smoke fixture custody --- .../opencti-known-digest-boundary.md | 16 +- .../stix-comparison-projection-boundary.md | 70 ++++++ src/stix_import.rs | 13 +- tests/opencti_hash_digest_boundary.rs | 147 +++++++++++- tests/smoke_admin_response.rs | 51 ++++- tests/smoke_process_lifecycle.rs | 31 ++- tests/stix_comparison_boundary.rs | 213 ++++++++++++++++++ tests/support/smoke_subtree.py | 147 ++++++++++++ tests/support/test_smoke_subtree.py | 98 ++++++++ 9 files changed, 771 insertions(+), 15 deletions(-) create mode 100644 docs/doctoring/stix-comparison-projection-boundary.md create mode 100644 tests/stix_comparison_boundary.rs create mode 100644 tests/support/smoke_subtree.py create mode 100644 tests/support/test_smoke_subtree.py diff --git a/docs/doctoring/opencti-known-digest-boundary.md b/docs/doctoring/opencti-known-digest-boundary.md index 813d8d00..eca9117a 100644 --- a/docs/doctoring/opencti-known-digest-boundary.md +++ b/docs/doctoring/opencti-known-digest-boundary.md @@ -44,13 +44,25 @@ repository: direct MD5; explicit array; explicit map after the array was fixed. Each repair was followed by the same real Axum control. The direct case pairs invalid rejection with valid-digest acceptance and an explicit block route. -Six integration tests additionally cover known lengths, short/long/nonhex/ +Eight integration tests additionally cover known lengths, short/long/nonhex/ UTF-8/internal-space denials, hyphenated algorithm spelling, uppercase and outer whitespace normalization, unknown-algorithm compatibility, mixed valid evidence, -unauthenticated denial, full management snapshot and persisted-byte nonmutation, +unauthenticated denial, five management projections and persisted-byte nonmutation, normal gateway admission, and fresh application reload. Fixtures allocate exclusive owned directories and remove only those directories. +The two follow-up tests characterize existing behavior; they do not repair a new +production defect. For direct, explicit-array and explicit-map MD5 documents, +missing and wrong tokens return HTTP401 and a read-only RBAC token returns +HTTP403 for both valid and invalid digests. A read-only read and an authorized +writer import provide positive controls. Every denial preserves the five +management projections and complete persisted bytes. For each of those three +input shapes with MD5/SHA1/SHA256/SHA512, fresh application loads preserve +normalized digest metadata and the exact feed source, counts, TTL and timestamp. +Invalid imports after reload preserve the same projections and persisted bytes +through a second load. These are sequential observations, not concurrent-read +or complete authentication-matrix guarantees. + These are synthetic documents through actual parser, management API, persistence and gateway code. They are not live OpenCTI pull, deployed authentication, file malware detection, complete STIX conformance, atomic concurrent-read guarantees, diff --git a/docs/doctoring/stix-comparison-projection-boundary.md b/docs/doctoring/stix-comparison-projection-boundary.md new file mode 100644 index 00000000..b4d893fe --- /dev/null +++ b/docs/doctoring/stix-comparison-projection-boundary.md @@ -0,0 +1,70 @@ +# STIX comparison projection boundary + +## Observed defect + +The shared STIX importer searched for the first `=` and then discarded the +property name. It projected `domain-name:value != 'status.example'` and +`domain-name:x_note = 'status.example'` into a positive domain indicator. + +A native gateway diagnostic used synthetic authorized documents on exclusive +loopback fixtures. Both STIX and OpenCTI imports returned HTTP201 for those +inputs instead of rejecting unsupported projection semantics. The saved critical +indicator caused score100/HTTP403 on the literal domain path and survived an +actual stop/restart. The two ordinary equality controls worked, and all six +unauthenticated imports returned HTTP401 without changing stored state. No +production incident or upstream service corruption was observed. + +The repository integration regression then observed two intended +HTTP201-versus400 failures before the source repair. The three-form success +control passed. This is separate from a private parser experiment, which was +not a shipped regression or complete HTTP acceptance. + +## Minimal shared repair + +`src/stix_import.rs` preserves the object/property distinction with +`split_once(':')`. For existing value-mapped IPv4, IPv6, domain, hostname and URL +types, the property must be exactly `value` after existing whitespace trimming. +The parser rejects operator remnants ending in `!`, `<` or `>` before the first +`=` and a second leading `=` on the right. This prevents `!=`, `<=`, `>=` and +malformed `==` from being projected as ordinary equality. `value NOT` is not +accepted as a literal value property either. + +The OASIS standard distinguishes equality and non-equality and defines object +paths as part of comparison expressions.[1] Its domain object has a distinct +`value` property.[1] These meanings justify retaining the property/operator +boundary; they do not prescribe Wardnet's limited projection policy or prove +complete conformance. Reference verification for this source change occurred +after initial implementation and the focused RED/GREEN. Prior receipt-only +retrieval chronology claims are excluded. + +The same parser is called by STIX import, OpenCTI pure-STIX delegation, OpenCTI +Indicator conversion and TAXII material conversion. There is no per-endpoint +replacement detector or new dependency. Unsupported-only input follows the +existing no-mappable response. Existing mixed-document skip behavior is not +changed into whole-document rejection. + +## Verified regression scope and limitations + +Three actual Axum tests cover direct STIX, OpenCTI pure-STIX and OpenCTI entities. +Across those forms they reject four operator variants and five unrelated or +modified property variants, preserve five management projections and complete +persisted bytes, reload the application and check score0/HTTP200 on the normal +gateway path. The ordinary uppercase-domain equality preserves normalized value, +source, severity and TTL across reload. + +This is a narrow repair. Unknown object/property projections, compound Boolean +patterns, qualifiers, escaped literals, unquoted-token compatibility, nested +bracket grammar and non-STIX pattern types are not comprehensively validated by +this change. File hash paths and general threat scoring are not redesigned. +It does not establish full STIX grammar, live TAXII/OpenCTI pulls, concurrent +persistence guarantees, deployed authentication, coverage100%, hosted security, +counted approval, normal merge or release. The native diagnostic's restart +observations do not expand the integration tests into crash-durability evidence. + +## Sources + +[1] STIX Version 2.1. Edited by Bret Jordan, Rich Piazza, and Trey Darley. +10 June 2021. OASIS Standard. Sections6.4.1,9.6,9.6.1. +https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html + +The reference is cited and summarized, not copied or redistributed as a PDF. diff --git a/src/stix_import.rs b/src/stix_import.rs index 7eefca23..fa2eac5a 100644 --- a/src/stix_import.rs +++ b/src/stix_import.rs @@ -196,8 +196,17 @@ fn parse_bracket_comparison(s: &str) -> Option<(String, String, usize)> { let eq = inner.find('=')?; let left = inner[..eq].trim(); let right = inner[eq + 1..].trim(); - let sco_type = left.split(':').next()?.trim(); - if sco_type.is_empty() { + let (sco_type, property) = left.split_once(':')?; + let sco_type = sco_type.trim(); + // Non-equality operators must not lose their meaning during projection. + if sco_type.is_empty() + || left.ends_with(['!', '<', '>']) + || right.starts_with('=') + || (matches!( + sco_type, + "ipv4-addr" | "ipv6-addr" | "domain-name" | "hostname" | "url" + ) && property.trim() != "value") + { return None; } let value = unquote_stix_string(right)?; diff --git a/tests/opencti_hash_digest_boundary.rs b/tests/opencti_hash_digest_boundary.rs index 5480262a..fda4f198 100644 --- a/tests/opencti_hash_digest_boundary.rs +++ b/tests/opencti_hash_digest_boundary.rs @@ -1,6 +1,7 @@ //! Synthetic export documents exercise actual management, storage and gateway code. //! No live OpenCTI service, operator credentials or file malware scanning. use std::{ + collections::HashMap, path::PathBuf, sync::atomic::{AtomicU64, Ordering}, }; @@ -11,7 +12,7 @@ use axum::{ }; use serde_json::{Value, json}; use tower::ServiceExt; -use waf_ids_ai_soc::{AppConfig, AppState, build_app}; +use waf_ids_ai_soc::{AdminPrincipal, AppConfig, AppState, build_app}; static FIXTURE_ID: AtomicU64 = AtomicU64::new(0); @@ -350,6 +351,150 @@ async fn app_after_reload(fixture: &StateFixture, token: &str) -> axum::Router { app(fixture, token).await } +#[tokio::test] +async fn every_digest_ingress_denies_wrong_and_readonly_tokens_before_digest_validation() { + for kind in ["direct", "array", "map"] { + let fixture = StateFixture::new(); + let writer = format!("opencti-writer-fixture-{}", std::process::id()); + let reader = format!("opencti-reader-fixture-{}", std::process::id()); + let wrong = format!("opencti-wrong-fixture-{}", std::process::id()); + drop(block_app(&fixture, &writer).await); + let state = AppState::load(AppConfig { + admin_token: None, + state_path: Some(fixture.path.clone()), + dnsbl_origin: "dnsbl.fixture".to_string(), + event_limit: 100, + }) + .await + .unwrap() + .with_admin_tokens(HashMap::from([ + ( + writer.clone(), + AdminPrincipal { + actor: "fixture-writer".into(), + can_write: true, + }, + ), + ( + reader.clone(), + AdminPrincipal { + actor: "fixture-reader".into(), + can_write: false, + }, + ), + ])); + let app = build_app(state); + assert_eq!( + read(&app, "/api/threats", &reader).await, + read(&app, "/api/threats", &writer).await + ); + allowed_status(&app).await; + let before = snapshot(&app, &writer).await; + let bytes = fixture.bytes(); + let valid = "AB".repeat(16); + for value in [valid.as_str(), "status"] { + let document = hash_document(kind, "MD5", value); + for (token, expected) in [ + (None, StatusCode::UNAUTHORIZED), + (Some(wrong.as_str()), StatusCode::UNAUTHORIZED), + (Some(reader.as_str()), StatusCode::FORBIDDEN), + ] { + let (status, body) = import(&app, &document, token).await; + assert_eq!(status, expected, "{kind}/{value}"); + let expected_error = if expected == StatusCode::FORBIDDEN { + "X-Admin-Token is not authorized for management writes" + } else { + "missing or invalid X-Admin-Token" + }; + assert_eq!(body["error"], expected_error); + assert_eq!(snapshot(&app, &writer).await, before); + assert_eq!(fixture.bytes(), bytes); + } + } + let (status, _) = import(&app, &hash_document(kind, "MD5", "status"), Some(&writer)).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(snapshot(&app, &writer).await, before); + assert_eq!(fixture.bytes(), bytes); + let (status, result) = + import(&app, &hash_document(kind, "MD5", &valid), Some(&writer)).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["upserted_threats"], 1); + assert_eq!(result["upserted_dnsbl"], 0); + let rows = read(&app, "/api/threats", &reader).await; + assert!( + rows.as_array() + .unwrap() + .iter() + .any(|row| row["source"] == "fixture:digest" + && row["value"] == valid.to_ascii_lowercase()) + ); + } +} + +#[tokio::test] +async fn every_known_digest_mapping_preserves_feed_metadata_and_rejection_after_reload() { + for kind in ["direct", "array", "map"] { + for (algorithm, length) in [("MD5", 32), ("SHA1", 40), ("SHA256", 64), ("SHA512", 128)] { + let fixture = StateFixture::new(); + let token = format!("opencti-digest-fixture-{}", std::process::id()); + let app = block_app(&fixture, &token).await; + let valid = "AB".repeat(length / 2); + let (status, result) = import( + &app, + &hash_document(kind, algorithm, &format!(" {valid} ")), + Some(&token), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "{kind}/{algorithm}"); + assert_eq!(result["feed_id"], "digest-boundary"); + assert_eq!(result["upserted_threats"], 1); + assert_eq!(result["upserted_dnsbl"], 0); + assert_eq!(result["skipped_objects"], 0); + assert!(result["last_updated_unix"].as_u64().unwrap() > 0); + let before = snapshot(&app, &token).await; + let rows: Vec<_> = before["threats"] + .as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == "fixture:digest") + .collect(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["value"], valid.to_ascii_lowercase()); + assert_eq!(rows[0]["indicator_type"], algorithm.to_ascii_lowercase()); + assert_eq!(rows[0]["severity"], "critical"); + assert_eq!(rows[0]["ttl_seconds"], 600); + let feeds: Vec<_> = before["feeds"] + .as_array() + .unwrap() + .iter() + .filter(|row| row["feed_id"] == "digest-boundary") + .collect(); + assert_eq!(feeds.len(), 1); + assert_eq!( + feeds[0], + &json!({"feed_id": "digest-boundary", "source": "fixture:digest", "ttl_seconds": 600, "threat_count": 1, "dnsbl_count": 0, "last_updated_unix": result["last_updated_unix"]}) + ); + let bytes = fixture.bytes(); + let reloaded = app_after_reload(&fixture, &token).await; + assert_eq!(snapshot(&reloaded, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + let (status, _) = import( + &reloaded, + &hash_document(kind, algorithm, "status"), + Some(&token), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{kind}/{algorithm}"); + assert_eq!(snapshot(&reloaded, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + let second_reload = app_after_reload(&fixture, &token).await; + assert_eq!(snapshot(&second_reload, &token).await, before); + assert_eq!(fixture.bytes(), bytes); + allowed_status(&second_reload).await; + } + } +} + #[tokio::test] async fn valid_direct_md5_preserves_digest_metadata_and_reload() { let fixture = StateFixture::new(); diff --git a/tests/smoke_admin_response.rs b/tests/smoke_admin_response.rs index 33ea7389..0c5ffc11 100644 --- a/tests/smoke_admin_response.rs +++ b/tests/smoke_admin_response.rs @@ -10,6 +10,7 @@ use std::{ net::TcpListener, path::PathBuf, process::{Command, Output, Stdio}, + sync::atomic::{AtomicU64, Ordering}, thread, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; @@ -17,6 +18,7 @@ use tower::ServiceExt; use waf_ids_ai_soc::{AppState, build_app}; const TITLE: &str = "ContextualWisdomLab WAF/IDS/AI SOC Gateway"; +static FIXTURE_ID: AtomicU64 = AtomicU64::new(0); fn actual_admin_check() -> &'static str { let script = include_str!("../scripts/smoke.sh"); @@ -29,16 +31,51 @@ struct FixtureDirectory(PathBuf); impl FixtureDirectory { fn new() -> Self { - let path = std::env::temp_dir().join(format!( - "wardnet smoke admin {} {}", - std::process::id(), + Self::new_at( SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() - .as_nanos() - )); - std::fs::create_dir(&path).unwrap(); - Self(path) + .as_nanos(), + ) + } + + fn new_at(timestamp: u128) -> Self { + for _ in 0..16 { + let path = std::env::temp_dir().join(format!( + "wardnet smoke admin {} {} {}", + std::process::id(), + timestamp, + FIXTURE_ID.fetch_add(1, Ordering::Relaxed) + )); + match std::fs::create_dir(&path) { + Ok(()) => return Self(path), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => panic!("create exclusive admin fixture: {error}"), + } + } + panic!("exclusive admin fixture names exhausted"); + } +} + +#[test] +fn equal_clock_admin_fixtures_have_independent_owned_directories() { + let fixtures: Vec<_> = (0..32).map(|_| FixtureDirectory::new_at(0)).collect(); + let paths: std::collections::HashSet<_> = + fixtures.iter().map(|fixture| fixture.0.clone()).collect(); + assert_eq!(paths.len(), fixtures.len()); + for (index, fixture) in fixtures.iter().enumerate() { + std::fs::write(fixture.0.join("sentinel"), index.to_string()).unwrap(); + } + let mut fixtures = fixtures; + let first = fixtures.remove(0); + let removed = first.0.clone(); + drop(first); + assert!(!removed.exists()); + for (index, fixture) in fixtures.iter().enumerate() { + assert_eq!( + std::fs::read_to_string(fixture.0.join("sentinel")).unwrap(), + (index + 1).to_string() + ); } } diff --git a/tests/smoke_process_lifecycle.rs b/tests/smoke_process_lifecycle.rs index 5919d1ad..fe6074ed 100644 --- a/tests/smoke_process_lifecycle.rs +++ b/tests/smoke_process_lifecycle.rs @@ -12,10 +12,12 @@ fn smoke_rejects_failed_or_ambiguous_cargo_artifact_discovery() { r#" import json, os, pathlib, subprocess, sys, tempfile script_path, scratch = map(pathlib.Path, sys.argv[1:]) +sys.path.insert(0, str(script_path.parent.parent / 'tests' / 'support')) +from smoke_subtree import owned_root, run as run_subtree script = script_path.read_text() prefix = script.split('\nstart_server\n', 1)[0] prefix = prefix.replace('ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"', 'ROOT_DIR="$OWNED_ROOT"') -with tempfile.TemporaryDirectory(prefix='wardnet cargo artifact ', dir=scratch) as directory: +with owned_root(prefix='wardnet cargo artifact ', dir=scratch) as directory: root = pathlib.Path(directory) tools = root / 'tools'; tools.mkdir() sentinel = tools / 'gateway' @@ -36,8 +38,8 @@ with tempfile.TemporaryDirectory(prefix='wardnet cargo artifact ', dir=scratch) 'ARTIFACT_BYTES': payload, 'ARTIFACT_EXIT': str(exit_code), 'EXECUTION_SENTINEL': str(root / 'executed')} driver = prefix + '\nstart_server\necho UNEXPECTED_START_SUCCESS\n' - result = subprocess.run(['bash', '-s'], input=driver.encode(), env=env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=6) + result = run_subtree(['bash', '-s'], input=driver.encode(), env=env, + timeout=6, root=root) assert result.returncode != 0, (name, 'failed discovery passed') assert not (root / 'executed').exists(), (name, 'binary executed before build acceptance') assert b'UNEXPECTED_START_SUCCESS' not in result.stdout, name @@ -56,6 +58,29 @@ with tempfile.TemporaryDirectory(prefix='wardnet cargo artifact ', dir=scratch) ); } +#[test] +#[cfg(unix)] +fn smoke_subtree_settlement_controls() { + let output = std::process::Command::new("python3") + .args([ + "-B", + concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/support/test_smoke_subtree.py" + ), + "-v", + ]) + .env("TMPDIR", std::env::temp_dir()) + .output() + .expect("run native subtree settlement controls"); + assert!( + output.status.success(), + "subtree controls failed: stdout={}, stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] #[cfg(unix)] fn smoke_owns_and_reaps_the_gateway_on_stop_restart_and_exit() { diff --git a/tests/stix_comparison_boundary.rs b/tests/stix_comparison_boundary.rs new file mode 100644 index 00000000..9c04be27 --- /dev/null +++ b/tests/stix_comparison_boundary.rs @@ -0,0 +1,213 @@ +//! Shared STIX comparison projection through actual management and persistence. +//! Synthetic documents/credentials only; no TAXII server or deployed incident. +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use std::{ + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppConfig, AppState, build_app}; + +static FIXTURE_ID: AtomicU64 = AtomicU64::new(0); +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "wardnet-stix-comparison-{}-{}", + std::process::id(), + FIXTURE_ID.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&root).unwrap(); + Self(root) + } + fn state(&self) -> PathBuf { + self.0.join("state.json") + } +} +impl Drop for Fixture { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).unwrap(); + } +} +async fn app(fixture: &Fixture) -> axum::Router { + build_app( + AppState::load(AppConfig { + admin_token: Some("synthetic-stix-token".into()), + state_path: Some(fixture.state()), + dnsbl_origin: "dnsbl.fixture".into(), + event_limit: 100, + }) + .await + .unwrap(), + ) +} +async fn request( + app: &axum::Router, + method: &str, + path: &str, + token: bool, + value: Option<&Value>, +) -> (StatusCode, Value) { + let mut builder = Request::builder().method(method).uri(path); + if token { + builder = builder.header("x-admin-token", "synthetic-stix-token"); + } + let body = match value { + Some(v) => { + builder = builder.header("content-type", "application/json"); + Body::from(v.to_string()) + } + None => Body::empty(), + }; + let response = app + .clone() + .oneshot(builder.body(body).unwrap()) + .await + .unwrap(); + let status = response.status(); + ( + status, + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap(), + ) +} +async fn snapshot(app: &axum::Router) -> Value { + let mut out = serde_json::Map::new(); + for path in [ + "/api/threats", + "/api/dnsbl", + "/api/threat-feeds", + "/api/audit-logs", + "/api/events", + ] { + let (status, body) = request(app, "GET", path, true, None).await; + assert_eq!(status, StatusCode::OK); + out.insert(path.into(), body); + } + Value::Object(out) +} +fn document(shape: &str, pattern: &str) -> Value { + let indicator = + json!({"type": "indicator", "pattern_type": "stix", "pattern": pattern, "confidence": 80}); + match shape { + "stix" | "opencti" => indicator, + "entities" => { + json!({"entities": [{"entity_type": "Indicator", "standard_id": "indicator--fixture", "pattern_type": "stix", "pattern": pattern, "confidence": 80}]}) + } + _ => panic!("unknown fixture shape"), + } +} +fn path(shape: &str) -> String { + let endpoint = if shape == "stix" { "stix" } else { "opencti" }; + format!( + "/api/threat-intel/{endpoint}?feed_id=comparison&source=fixture:comparison&ttl_seconds=600" + ) +} +async fn rejection(shape: &str, pattern: &str) { + let fixture = Fixture::new(); + let app = app(&fixture).await; + let route = json!({"id":"comparison", "path_prefix":"/comparison", "upstream":"mock://fixture", "mode":"block", "enabled":true}); + assert_eq!( + request(&app, "POST", "/api/routes", true, Some(&route)) + .await + .0, + StatusCode::CREATED + ); + let before = snapshot(&app).await; + let bytes = std::fs::read(fixture.state()).unwrap(); + let doc = document(shape, pattern); + assert_eq!( + request(&app, "POST", &path(shape), false, Some(&doc)) + .await + .0, + StatusCode::UNAUTHORIZED + ); + assert_eq!(snapshot(&app).await, before); + assert_eq!(std::fs::read(fixture.state()).unwrap(), bytes); + let (status, _) = request(&app, "POST", &path(shape), true, Some(&doc)).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "{shape}: unsupported comparison projected into a positive indicator: {pattern}" + ); + assert_eq!(snapshot(&app).await, before); + assert_eq!(std::fs::read(fixture.state()).unwrap(), bytes); + let reloaded = self::app(&fixture).await; + assert_eq!(snapshot(&reloaded).await, before); + let (status, body) = request( + &reloaded, + "GET", + "/gateway/comparison/status.example", + false, + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["score"], 0); +} +#[tokio::test] +async fn non_equality_does_not_project_a_positive_domain_indicator() { + for shape in ["stix", "opencti", "entities"] { + for operator in ["!=", ">=", "<=", "=="] { + rejection( + shape, + &format!("[domain-name:value {operator} 'status.example']"), + ) + .await; + } + } +} +#[tokio::test] +async fn unrelated_property_does_not_project_the_domain_value() { + for shape in ["stix", "opencti", "entities"] { + for property in [ + "x_note", + "resolves_to_refs", + "value NOT", + "value!", + "value.other", + ] { + rejection( + shape, + &format!("[domain-name:{property} = 'status.example']"), + ) + .await; + } + } +} +#[tokio::test] +async fn literal_value_equality_preserves_authorized_rows_and_reload() { + for shape in ["stix", "opencti", "entities"] { + let fixture = Fixture::new(); + let app = app(&fixture).await; + let pattern = "[domain-name:value = 'STATUS.EXAMPLE']"; + let (status, result) = request( + &app, + "POST", + &path(shape), + true, + Some(&document(shape, pattern)), + ) + .await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(result["upserted_threats"], 1); + assert_eq!(result["upserted_dnsbl"], 0); + let before = snapshot(&app).await; + let rows: Vec<_> = before["/api/threats"] + .as_array() + .unwrap() + .iter() + .filter(|r| r["source"] == "fixture:comparison") + .collect(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0], + &json!({"value":"status.example","indicator_type":"domain","severity":"critical","source":"fixture:comparison","ttl_seconds":600}) + ); + let reloaded = self::app(&fixture).await; + assert_eq!(snapshot(&reloaded).await, before); + } +} diff --git a/tests/support/smoke_subtree.py b/tests/support/smoke_subtree.py new file mode 100644 index 00000000..0195e302 --- /dev/null +++ b/tests/support/smoke_subtree.py @@ -0,0 +1,147 @@ +"""Finite same-process-group custody for POSIX offline smoke fixtures. + +Settlement means an exact supervisor wait and an empty group observation, not +physical grandchild waits or containment of children that deliberately setsid. +Permanent OS failure and interruption during an unreturned native Popen spawn +are outside this cooperative fixture contract. Uncertainty retains the root. +""" +import contextlib +import os +import selectors +import shutil +import signal +import subprocess +import tempfile +import time + + +class SettlementError(RuntimeError): + """The caller must retain its fixture because settlement is unproved.""" + + +# Keep the returned owner on uncertainty and refuse further fixture launches. +_unresolved = [] + + +@contextlib.contextmanager +def owned_root(*, prefix, dir): + """Dispose only an exclusively created root with established settlement.""" + root = tempfile.mkdtemp(prefix=prefix, dir=dir) + retain = False + try: + yield root + except SettlementError: + retain = True + raise + finally: + if not retain and not _unresolved: + shutil.rmtree(root) + + +def members(group): + output = subprocess.check_output( + ['ps', '-axo', 'pid,ppid,pgid'], text=True, timeout=3 + ) + rows = [] + for line in output.splitlines()[1:]: + fields = line.split() + if len(fields) != 3 or not all(field.isdigit() for field in fields): + raise SettlementError('invalid process observation') + if int(fields[2]) == group: + rows.append(int(fields[0])) + return rows + + +def run(args, *, input, env, timeout, root): + """Preserve the primary error after bounded same-group settlement.""" + if _unresolved: + raise SettlementError('previous fixture unsettled; no new launch') + assert args == ['bash', '-s'] and timeout > 0 + child = None + primary = None + settled = False + result = None + with contextlib.ExitStack() as resources: + # Register returned descriptors before further fallible acquisitions. + fds = [] + for _ in range(2): + for fd in os.pipe(): + fds.append(fd) + resources.callback(os.close, fd) + status_read, status_write, release_read, release_write = fds + source = resources.enter_context(tempfile.TemporaryFile(dir=root)) + stdout = resources.enter_context(tempfile.TemporaryFile(dir=root)) + stderr = resources.enter_context(tempfile.TemporaryFile(dir=root)) + source.write(input) + source.seek(0) + wrapper = ''' +trap ':' TERM +bash -s +result=$? +trap '' TERM +printf '%s\\n' "$result" >&"$1" +IFS= read -r -t 15 release <&"$2" +exit "$result" +''' + try: + child = subprocess.Popen( + ['bash', '-c', wrapper, 'smoke-owner', str(status_write), str(release_read)], + stdin=source, stdout=stdout, stderr=stderr, env=env, + pass_fds=(status_write, release_read), start_new_session=True, + ) + deadline = time.monotonic() + timeout + with selectors.DefaultSelector() as selector: + selector.register(status_read, selectors.EVENT_READ) + if not selector.select(max(0, deadline - time.monotonic())): + raise subprocess.TimeoutExpired(args, timeout) + os.set_blocking(status_read, False) + raw = os.read(status_read, 32) + if not raw.endswith(b'\n') or not raw[:-1].isdigit(): + raise SettlementError('invalid supervisor result') + result = int(raw[:-1]) + except BaseException as exc: + primary = exc + finally: + cleanup_error = None + if child is not None: + try: + # The leader pins the group until release or escalation. + # There are no signals after the exact wait below. + os.killpg(child.pid, signal.SIGTERM) + deadline = time.monotonic() + 3 + remaining = members(child.pid) + while any(pid != child.pid for pid in remaining) and time.monotonic() < deadline: + time.sleep(.01) + remaining = members(child.pid) + if any(pid != child.pid for pid in remaining): + os.killpg(child.pid, signal.SIGKILL) + else: + os.write(release_write, b'release\n') + except BaseException as exc: + cleanup_error = exc + # Signal/observation failure cannot skip an exact wait attempt. + try: + child.wait(timeout=3) + deadline = time.monotonic() + 3 + while members(child.pid) and time.monotonic() < deadline: + time.sleep(.01) + if members(child.pid): + raise SettlementError('subtree settlement unknown') + if cleanup_error is None: + settled = True + except BaseException as exc: + if cleanup_error is None: + cleanup_error = exc + if not settled: + _unresolved.append(child) + else: + settled = True + if cleanup_error is not None and primary is None: + primary = cleanup_error + if not settled: + raise SettlementError('subtree settlement unknown; retain fixture') from primary + if primary is not None: + raise primary + stdout.seek(0) + stderr.seek(0) + return subprocess.CompletedProcess(args, result, stdout.read(), stderr.read()) diff --git a/tests/support/test_smoke_subtree.py b/tests/support/test_smoke_subtree.py new file mode 100644 index 00000000..df5e5490 --- /dev/null +++ b/tests/support/test_smoke_subtree.py @@ -0,0 +1,98 @@ +"""Native finite fixture controls, not arbitrary process escape containment.""" +import os +import pathlib +import shutil +import shlex +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import smoke_subtree + + +class SmokeSubtreeTests(unittest.TestCase): + def setUp(self): + self.root = pathlib.Path(tempfile.mkdtemp(prefix='wardnet-subtree-', dir=os.environ.get('TMPDIR'))) + self.env = {'PATH': os.defpath, 'TMPDIR': str(self.root)} + self.addCleanup(shutil.rmtree, self.root) + + def run_shell(self, script, timeout=6.0): + return smoke_subtree.run(['bash', '-s'], input=script.encode(), env=self.env, + timeout=timeout, root=self.root) + + def test_normal_nonzero_output_parity(self): + result = self.run_shell("printf out; printf err >&2; exit 42\n") + self.assertEqual((result.returncode, result.stdout, result.stderr), (42, b'out', b'err')) + + def test_timeout_propagates_after_settlement(self): + with self.assertRaises(subprocess.TimeoutExpired) as raised: + self.run_shell('sleep 30\n', timeout=.2) + self.assertEqual(raised.exception.timeout, .2) + self.assertEqual(smoke_subtree._unresolved, []) + + def test_leader_first_background_child_is_settled(self): + marker = self.root / 'child' + result = self.run_shell(f'sleep 30 &\nprintf "%s" "$!" > "{marker}"\nexit 0\n') + self.assertEqual(result.returncode, 0) + self.assert_absent(int(marker.read_text())) + + def test_term_ignoring_background_child_requires_escalation(self): + marker = self.root / 'child' + actor = self.root / 'ignore_term.py' + actor.write_text('import os,pathlib,signal,sys,time\nsignal.signal(signal.SIGTERM,signal.SIG_IGN)\npathlib.Path(sys.argv[1]).write_text(str(os.getpid()))\ntime.sleep(30)\n') + result = self.run_shell(f'{shlex.quote(sys.executable)} {shlex.quote(str(actor))} {shlex.quote(str(marker))} &\nwhile [ ! -s "{marker}" ]; do sleep .01; done\nexit 0\n') + self.assertEqual(result.returncode, 0) + self.assert_absent(int(marker.read_text())) + + def assert_absent(self, pid): + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_unknown_settlement_retains_fixture(self): + retained = None + try: + with self.assertRaises(smoke_subtree.SettlementError): + with smoke_subtree.owned_root(prefix='wardnet-retain-', dir=self.root) as root: + retained = pathlib.Path(root) + raise smoke_subtree.SettlementError('synthetic uncertainty') + assert retained is not None + self.assertTrue(retained.is_dir()) + finally: + if retained is not None: + shutil.rmtree(retained) + + def test_normal_fixture_is_removed(self): + with smoke_subtree.owned_root(prefix='wardnet-remove-', dir=self.root) as root: + created = pathlib.Path(root) + self.assertTrue(created.is_dir()) + self.assertFalse(created.exists()) + + def test_signal_failure_still_attempts_wait_and_fences_next_launch(self): + class Child: + pid = 123456789 + waited = False + def wait(self, timeout): + self.waited = True + return 0 + child = Child() + previous = list(smoke_subtree._unresolved) + try: + with patch.object(smoke_subtree.subprocess, 'Popen', return_value=child), \ + patch.object(smoke_subtree.selectors, 'DefaultSelector', side_effect=RuntimeError('primary')), \ + patch.object(smoke_subtree.os, 'killpg', side_effect=PermissionError('signal')), \ + patch.object(smoke_subtree, 'members', return_value=[]): + with self.assertRaises(smoke_subtree.SettlementError) as raised: + self.run_shell('exit 0\n') + self.assertIsInstance(raised.exception.__cause__, RuntimeError) + self.assertTrue(child.waited) + self.assertEqual(smoke_subtree._unresolved, [child]) + with self.assertRaises(smoke_subtree.SettlementError): + self.run_shell('exit 0\n') + finally: + smoke_subtree._unresolved[:] = previous + + +if __name__ == '__main__': + unittest.main() From 931e24de7ed2f8881ca1871917efef48d0fa36ef Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Tue, 6 Oct 2026 17:51:42 +0900 Subject: [PATCH 20/22] fix(stix): do not admit revoked indicators as enforcement evidence The shared STIX parser (STIX, TAXII poll, OpenCTI) now admits an indicator only when revoked is absent or false; the OpenCTI entity rebuild keeps the field. The artifact-discovery smoke test runs python3 -B so the suite no longer writes bytecode into the checkout. --- .../stix-revoked-indicator-boundary.md | 67 ++++++ src/opencti_import.rs | 1 + src/stix_import.rs | 9 + tests/smoke_process_lifecycle.rs | 1 + tests/stix_revoked_indicator_boundary.rs | 211 ++++++++++++++++++ 5 files changed, 289 insertions(+) create mode 100644 docs/doctoring/stix-revoked-indicator-boundary.md create mode 100644 tests/stix_revoked_indicator_boundary.rs diff --git a/docs/doctoring/stix-revoked-indicator-boundary.md b/docs/doctoring/stix-revoked-indicator-boundary.md new file mode 100644 index 00000000..d0674abb --- /dev/null +++ b/docs/doctoring/stix-revoked-indicator-boundary.md @@ -0,0 +1,67 @@ +# STIX revoked indicator boundary + +## Observed defect + +The shared STIX importer (`src/stix_import.rs`) never read the STIX common property +`revoked`. An indicator with `"revoked": true` and pattern +`[ipv4-addr:value = '203.0.113.66']` was projected into an enforceable critical `client_ip` +threat row plus a DNSBL `127.0.0.2` entry. The rows were persisted, survived reload and made +the gateway return HTTP403 for that client on a block-mode route. + +The same parser serves three ingress paths: + +- `POST /api/threat-intel/stix` (bundle, indicator or array); +- `POST /api/threat-intel/taxii/poll` (TAXII 2.1 envelope normalized to a STIX bundle); +- `POST /api/threat-intel/opencti` for native STIX indicators and OpenCTI `Indicator` + entities. + +For the OpenCTI entity shape, `materialize_node` (`src/opencti_import.rs`) rebuilt a fresh +indicator object and dropped `revoked` before the STIX parser ran. + +## Standard + +OASIS, *STIX Version 2.1*, OASIS Standard, 10 June 2021, section 3.2 (common properties), +property `revoked` (boolean): "Revoked objects are no longer considered valid by the object +creator. Revoking an object is permanent ... The default value of this property is false." +Section 3.6 adds: "This specification does not address how implementations should handle +revoked data." + +Wardnet's policy decision is therefore explicit, not inherited from the standard: an +enforcement gateway must not admit evidence that its creator has declared invalid. + +## Actual RED + +`tests/stix_revoked_indicator_boundary.rs` drives the real Axum app with synthetic +credentials and a temporary state file, across the `stix` bundle, OpenCTI native-indicator +array and OpenCTI entity shapes: + +- mixed bundle: one `revoked: true`, one `revoked: false`, one with `revoked` absent; +- all-revoked and malformed (`"false"`, `1`, `null`) documents, each with an unauthenticated + 401/no-mutation control first. + +Before the repair, `cargo test --locked --test stix_revoked_indicator_boundary` failed both +tests (exit101): the mixed bundle reported `upserted_threats: 3, upserted_dnsbl: 3`, and the +all-revoked document returned HTTP201 instead of HTTP400. + +## Repair + +- `src/stix_import.rs`: after the `type == indicator` check, an object is admitted only when + `revoked` is absent or exactly the JSON boolean `false`. Any other value is counted in + `skipped_objects` (fail closed for malformed lifecycle data). +- `src/opencti_import.rs`: the synthesized entity-shape indicator carries the node's `revoked` + value (default `false`). + +After the repair, the mixed bundle stores and enforces only the two live indicators (HTTP403 +for them after reload), the revoked address scores 0/HTTP200, and all-revoked or malformed +documents return HTTP400 with byte-identical persisted state. + +## Limits + +- This stops new admission of revoked indicators. It does not retract rows previously + imported under the same STIX id. Same-feed refresh reconciliation and DNSBL snapshot + ownership remain with issue #172. +- STIX version ordering by `modified`, `valid_from`/`valid_until` windows and non-STIX + `pattern_type` values are not handled by this slice. +- The reputation `EvidenceRecordV1` lifecycle in issue #184 is a separate, unmerged code + path. This repair is related but does not close that issue. +- Synthetic local fixtures only: no live TAXII server, OpenCTI instance or deployed incident. diff --git a/src/opencti_import.rs b/src/opencti_import.rs index 1b9bee21..d7bb2db4 100644 --- a/src/opencti_import.rs +++ b/src/opencti_import.rs @@ -250,6 +250,7 @@ fn materialize_node(node: &serde_json::Value, source: &str, ttl_seconds: u64) -> "name": node.get("name").cloned().unwrap_or(serde_json::json!("opencti-indicator")), "pattern": pattern, "pattern_type": node.get("pattern_type").cloned().unwrap_or(serde_json::json!("stix")), + "revoked": node.get("revoked").cloned().unwrap_or(serde_json::json!(false)), "valid_from": "1970-01-01T00:00:00Z", "confidence": node.get("confidence") .or_else(|| node.get("x_opencti_score")) diff --git a/src/stix_import.rs b/src/stix_import.rs index fa2eac5a..c3846542 100644 --- a/src/stix_import.rs +++ b/src/stix_import.rs @@ -56,6 +56,15 @@ pub fn stix_material_from_value( skipped_objects += 1; continue; } + // STIX 2.1 section 3.2: revoked objects are no longer valid for their creator. + // Only an absent or literal `false` value is admissible enforcement evidence. + if !matches!( + obj.get("revoked"), + None | Some(serde_json::Value::Bool(false)) + ) { + skipped_objects += 1; + continue; + } let pattern = obj .get("pattern") .and_then(|p| p.as_str()) diff --git a/tests/smoke_process_lifecycle.rs b/tests/smoke_process_lifecycle.rs index fe6074ed..6ed8120e 100644 --- a/tests/smoke_process_lifecycle.rs +++ b/tests/smoke_process_lifecycle.rs @@ -8,6 +8,7 @@ fn smoke_rejects_failed_or_ambiguous_cargo_artifact_discovery() { let output = Command::new("python3") .args([ + "-B", "-c", r#" import json, os, pathlib, subprocess, sys, tempfile diff --git a/tests/stix_revoked_indicator_boundary.rs b/tests/stix_revoked_indicator_boundary.rs new file mode 100644 index 00000000..4a4d6f15 --- /dev/null +++ b/tests/stix_revoked_indicator_boundary.rs @@ -0,0 +1,211 @@ +//! STIX `revoked` lifecycle boundary through actual management, persistence and gateway. +//! OASIS STIX 2.1 (10 June 2021) section 3.2 common property `revoked`: "Revoked objects are +//! no longer considered valid by the object creator." The specification leaves handling to +//! implementations; Wardnet does not admit revoked indicators as enforcement evidence. +//! Synthetic documents/credentials only; no TAXII server or deployed incident. +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::{Value, json}; +use std::{ + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppConfig, AppState, build_app}; + +static FIXTURE_ID: AtomicU64 = AtomicU64::new(0); +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "wardnet-stix-revoked-{}-{}", + std::process::id(), + FIXTURE_ID.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&root).unwrap(); + Self(root) + } + fn state(&self) -> PathBuf { + self.0.join("state.json") + } +} +impl Drop for Fixture { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).unwrap(); + } +} +async fn app(fixture: &Fixture) -> axum::Router { + build_app( + AppState::load(AppConfig { + admin_token: Some("synthetic-revoked-token".into()), + state_path: Some(fixture.state()), + dnsbl_origin: "dnsbl.fixture".into(), + event_limit: 100, + }) + .await + .unwrap(), + ) +} +async fn request( + app: &axum::Router, + method: &str, + path: &str, + token: bool, + client: Option<&str>, + value: Option<&Value>, +) -> (StatusCode, Value) { + let mut builder = Request::builder().method(method).uri(path); + if token { + builder = builder.header("x-admin-token", "synthetic-revoked-token"); + } + if let Some(ip) = client { + builder = builder.header("x-forwarded-for", ip); + } + let body = match value { + Some(v) => { + builder = builder.header("content-type", "application/json"); + Body::from(v.to_string()) + } + None => Body::empty(), + }; + let response = app + .clone() + .oneshot(builder.body(body).unwrap()) + .await + .unwrap(); + let status = response.status(); + ( + status, + serde_json::from_slice(&to_bytes(response.into_body(), 1_000_000).await.unwrap()).unwrap(), + ) +} +async fn snapshot(app: &axum::Router) -> Value { + let mut out = serde_json::Map::new(); + for path in [ + "/api/threats", + "/api/dnsbl", + "/api/threat-feeds", + "/api/audit-logs", + "/api/events", + ] { + let (status, body) = request(app, "GET", path, true, None, None).await; + assert_eq!(status, StatusCode::OK); + out.insert(path.into(), body); + } + Value::Object(out) +} +fn indicator(shape: &str, ip: &str, revoked: Option) -> Value { + let pattern = format!("[ipv4-addr:value = '{ip}']"); + let mut object = if shape == "entities" { + json!({"entity_type": "Indicator", "standard_id": format!("indicator--{ip}"), "pattern_type": "stix", "pattern": pattern, "confidence": 80}) + } else { + json!({"type": "indicator", "id": format!("indicator--{ip}"), "pattern_type": "stix", "pattern": pattern, "confidence": 80}) + }; + if let Some(flag) = revoked { + object["revoked"] = flag; + } + object +} +fn document(shape: &str, objects: Vec) -> Value { + match shape { + "stix" => json!({"type": "bundle", "id": "bundle--revoked-fixture", "objects": objects}), + "opencti" => Value::Array(objects), + "entities" => json!({"entities": objects}), + _ => panic!("unknown fixture shape"), + } +} +fn path(shape: &str) -> String { + let endpoint = if shape == "stix" { "stix" } else { "opencti" }; + format!("/api/threat-intel/{endpoint}?feed_id=revoked&source=fixture:revoked&ttl_seconds=600") +} +fn values(body: &Value, field: &str) -> Vec { + body.as_array() + .unwrap() + .iter() + .filter(|row| row["source"] == "fixture:revoked") + .map(|row| row[field].as_str().unwrap().to_string()) + .collect() +} +async fn block_route(app: &axum::Router) { + let route = json!({"id":"revoked", "path_prefix":"/revoked", "upstream":"mock://fixture", "mode":"block", "enabled":true}); + let (status, _) = request(app, "POST", "/api/routes", true, None, Some(&route)).await; + assert_eq!(status, StatusCode::CREATED); +} +async fn gateway_score(app: &axum::Router, ip: &str) -> (StatusCode, Value) { + request(app, "GET", "/gateway/revoked/x", false, Some(ip), None).await +} + +#[tokio::test] +async fn revoked_indicator_is_not_admitted_while_live_siblings_are_enforced() { + for shape in ["stix", "opencti", "entities"] { + let fixture = Fixture::new(); + let app = app(&fixture).await; + block_route(&app).await; + let doc = document( + shape, + vec![ + indicator(shape, "203.0.113.66", Some(json!(true))), + indicator(shape, "203.0.113.10", Some(json!(false))), + indicator(shape, "203.0.113.11", None), + ], + ); + let (status, result) = request(&app, "POST", &path(shape), true, None, Some(&doc)).await; + assert_eq!(status, StatusCode::CREATED, "{shape}: {result}"); + assert_eq!(result["upserted_threats"], 2, "{shape}: {result}"); + assert_eq!(result["upserted_dnsbl"], 2, "{shape}: {result}"); + let state = snapshot(&app).await; + let mut threats = values(&state["/api/threats"], "value"); + threats.sort(); + assert_eq!(threats, ["203.0.113.10", "203.0.113.11"], "{shape}"); + let mut dnsbl = values(&state["/api/dnsbl"], "address"); + dnsbl.sort(); + assert_eq!(dnsbl, ["203.0.113.10", "203.0.113.11"], "{shape}"); + + let reloaded = self::app(&fixture).await; + assert_eq!(snapshot(&reloaded).await, state, "{shape}: reload"); + let (status, body) = gateway_score(&reloaded, "203.0.113.66").await; + assert_eq!( + status, + StatusCode::OK, + "{shape}: revoked evidence blocked: {body}" + ); + assert_eq!(body["score"], 0, "{shape}: {body}"); + for live in ["203.0.113.10", "203.0.113.11"] { + let (status, _) = gateway_score(&reloaded, live).await; + assert_eq!(status, StatusCode::FORBIDDEN, "{shape}: live {live}"); + } + } +} + +#[tokio::test] +async fn all_revoked_or_malformed_revocation_rejects_without_mutation() { + for shape in ["stix", "opencti", "entities"] { + for flag in [json!(true), json!("false"), json!(1), json!(null)] { + let fixture = Fixture::new(); + let app = app(&fixture).await; + block_route(&app).await; + let before = snapshot(&app).await; + let bytes = std::fs::read(fixture.state()).unwrap(); + let doc = document( + shape, + vec![indicator(shape, "203.0.113.66", Some(flag.clone()))], + ); + let (status, _) = request(&app, "POST", &path(shape), false, None, Some(&doc)).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(snapshot(&app).await, before); + let (status, body) = request(&app, "POST", &path(shape), true, None, Some(&doc)).await; + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "{shape}: revoked={flag} admitted as enforcement evidence: {body}" + ); + assert_eq!(snapshot(&app).await, before, "{shape}: revoked={flag}"); + assert_eq!(std::fs::read(fixture.state()).unwrap(), bytes); + let (status, body) = gateway_score(&app, "203.0.113.66").await; + assert_eq!(status, StatusCode::OK, "{shape}: revoked={flag}"); + assert_eq!(body["score"], 0); + } + } +} From 50697970a4a078dd1c6d2c2fe50276626050312e Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Tue, 6 Oct 2026 22:49:59 +0900 Subject: [PATCH 21/22] test: cover TAXII/Coraza/DNSBL boundaries and load-proof smoke custody bounds Adds unit tests for TAXII input validation, Coraza NDJSON/fallbacks and the shared-owner shortest DNSBL TTL. Smoke fixture hang guards were shorter than measured run time under host load (3-7 s per case, ps probe >3 s); they are now finite but load-tolerant, with unchanged TERM/KILL settlement semantics. --- crates/waf-ids-core/src/lib.rs | 27 +++++++++++++++++++ src/coraza_audit.rs | 33 +++++++++++++++++++++++ src/taxii.rs | 45 ++++++++++++++++++++++++++++++++ tests/smoke_process_lifecycle.rs | 4 ++- tests/support/smoke_subtree.py | 16 ++++++++---- 5 files changed, 119 insertions(+), 6 deletions(-) diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index e5b4fadb..ff8baf67 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -1579,6 +1579,33 @@ mod dnsbl_txt; mod tests { use super::*; + /// Two publishable entries for the same IPv4 owner share one TTL per RRset + /// (RFC 2181 section 5.2): the shortest lifetime wins regardless of order. + #[test] + fn dnsbl_shared_owner_uses_shortest_ttl_in_both_orders() { + let mut long = AppData::seeded().dnsbl.remove(0); + long.address = "192.0.2.77".parse().unwrap(); + long.ttl_seconds = 900; + long.source = "unit-long".into(); + let mut short = long.clone(); + short.ttl_seconds = 120; + short.source = "unit-short".into(); + for entries in [[long.clone(), short.clone()], [short, long]] { + let zone = export_dnsbl_zone("dnsbl.example", &entries); + let owner_lines: Vec<_> = zone + .lines() + .filter(|line| line.starts_with("77.2.0.192 ")) + .collect(); + assert_eq!(owner_lines.len(), 4, "{zone}"); + assert!( + owner_lines + .iter() + .all(|line| line.starts_with("77.2.0.192 120 IN ")), + "{zone}" + ); + } + } + /// Exercise all new RDATA accounting branches in the unit-library object. /// These are coverage controls; actual admission/export REDs are retained. #[test] diff --git a/src/coraza_audit.rs b/src/coraza_audit.rs index 362460ae..4ed97dac 100644 --- a/src/coraza_audit.rs +++ b/src/coraza_audit.rs @@ -266,6 +266,39 @@ mod tests { assert_eq!(parsed.hits[0].path, "/admin"); } + #[test] + fn ndjson_skips_blank_lines_and_reports_invalid_line_number() { + let body = "{\"transaction\":{\"is_interrupted\":true}}\n\n \n{\"transaction\":{\"is_interrupted\":true}}\n"; + let parsed = parse_coraza_audit_body(body).unwrap(); + assert_eq!(parsed.hits.len(), 2); + assert_eq!(parsed.skipped, 0); + let error = parse_coraza_audit_body("{\"a\":1}\n\nnot-json\n").unwrap_err(); + assert!( + error.starts_with("invalid Coraza audit JSON on line 3:"), + "{error}" + ); + } + + #[test] + fn reason_and_message_fallbacks_are_stable() { + let hit = |raw: &str| coraza_hit_from_value(&serde_json::from_str(raw).unwrap()).unwrap(); + // Rule id and severity directly on the message, empty text. + let only_id = hit(r#"{"messages":[{"id":941100,"severity":5}]}"#); + assert_eq!(only_id.reason, "coraza/crs: rule 941100"); + assert_eq!(only_id.score, 25); + assert_eq!(only_id.action, "monitor"); + assert_eq!(only_id.path, "coraza://transaction"); + // Text from data.msg without a rule id. + let data_msg = hit(r#"{"messages":[{"data":{"msg":"Scanner detected","severity":0}}]}"#); + assert_eq!(data_msg.reason, "coraza/crs: Scanner detected"); + assert_eq!(data_msg.score, 80); + assert_eq!(data_msg.action, "block"); + // Text from a top-level msg on the message object. + let first_msg = hit(r#"{"messages":[{"data":{},"msg":"Generic hit"}]}"#); + assert_eq!(first_msg.reason, "coraza/crs: Generic hit"); + assert_eq!(first_msg.score, 50); + } + #[test] fn rejects_empty_body() { assert!(parse_coraza_audit_body(" \n").is_err()); diff --git a/src/taxii.rs b/src/taxii.rs index 775e23b2..82c87106 100644 --- a/src/taxii.rs +++ b/src/taxii.rs @@ -146,4 +146,49 @@ mod tests { assert!(url.contains("added_after=")); assert!(url.contains("2024")); } + + #[test] + fn rejects_empty_or_unsafe_collection_inputs() { + assert_eq!( + collection_objects_url(" / ", "abc").unwrap_err(), + "api_root must be non-empty" + ); + assert_eq!( + collection_objects_url("https://taxii.example/api1", " / ").unwrap_err(), + "collection_id must be non-empty" + ); + for id in ["a?b", "a#b"] { + assert!(collection_objects_url("https://taxii.example/api1", id).is_err()); + } + } + + #[test] + fn filter_boundary_rejects_unsafe_values_and_keeps_plain_urls() { + assert_eq!( + with_taxii_filters(" ", None).unwrap_err(), + "objects_url must be non-empty" + ); + let base = "https://taxii.example/api1/collections/c/objects/"; + assert_eq!(with_taxii_filters(base, None).unwrap(), base); + assert_eq!(with_taxii_filters(base, Some(" ")).unwrap(), base); + for after in ["2024&x=1", "2024#frag", "20\n24", "20\u{7f}24"] { + assert_eq!( + with_taxii_filters(base, Some(after)).unwrap_err(), + "added_after contains invalid characters" + ); + } + assert!( + with_taxii_filters("not a url", Some("2024-01-01T00:00:00Z")) + .unwrap_err() + .starts_with("invalid objects_url:") + ); + } + + #[test] + fn passes_through_indicator_and_array_unchanged() { + let indicator = r#"{"type":"indicator","id":"indicator--1"}"#; + assert_eq!(stix_json_from_taxii_response(indicator).unwrap(), indicator); + let array = r#"[{"type":"indicator"}]"#; + assert_eq!(stix_json_from_taxii_response(array).unwrap(), array); + } } diff --git a/tests/smoke_process_lifecycle.rs b/tests/smoke_process_lifecycle.rs index 6ed8120e..20abba99 100644 --- a/tests/smoke_process_lifecycle.rs +++ b/tests/smoke_process_lifecycle.rs @@ -39,8 +39,10 @@ with owned_root(prefix='wardnet cargo artifact ', dir=scratch) as directory: 'ARTIFACT_BYTES': payload, 'ARTIFACT_EXIT': str(exit_code), 'EXECUTION_SENTINEL': str(root / 'executed')} driver = prefix + '\nstart_server\necho UNEXPECTED_START_SUCCESS\n' + # Hang guard only: each case starts several python3 processes, and on a + # loaded host one case measured 3-7 s wall time, so 6 s was load-flaky. result = run_subtree(['bash', '-s'], input=driver.encode(), env=env, - timeout=6, root=root) + timeout=60, root=root) assert result.returncode != 0, (name, 'failed discovery passed') assert not (root / 'executed').exists(), (name, 'binary executed before build acceptance') assert b'UNEXPECTED_START_SUCCESS' not in result.stdout, name diff --git a/tests/support/smoke_subtree.py b/tests/support/smoke_subtree.py index 0195e302..fb1bf6bc 100644 --- a/tests/support/smoke_subtree.py +++ b/tests/support/smoke_subtree.py @@ -19,6 +19,12 @@ class SettlementError(RuntimeError): """The caller must retain its fixture because settlement is unproved.""" +# Finite hang guards, sized for heavily loaded shared hosts. A 3 s `ps` probe +# and 3 s settlement windows were observed to expire at load average ~170 on +# 10 CPUs; a slow observation is not evidence of an escaped subtree. +OBSERVE_TIMEOUT = 30 +SETTLE_SECONDS = 15 + # Keep the returned owner on uncertainty and refuse further fixture launches. _unresolved = [] @@ -40,7 +46,7 @@ def owned_root(*, prefix, dir): def members(group): output = subprocess.check_output( - ['ps', '-axo', 'pid,ppid,pgid'], text=True, timeout=3 + ['ps', '-axo', 'pid,ppid,pgid'], text=True, timeout=OBSERVE_TIMEOUT ) rows = [] for line in output.splitlines()[1:]: @@ -80,7 +86,7 @@ def run(args, *, input, env, timeout, root): result=$? trap '' TERM printf '%s\\n' "$result" >&"$1" -IFS= read -r -t 15 release <&"$2" +IFS= read -r -t 60 release <&"$2" exit "$result" ''' try: @@ -108,7 +114,7 @@ def run(args, *, input, env, timeout, root): # The leader pins the group until release or escalation. # There are no signals after the exact wait below. os.killpg(child.pid, signal.SIGTERM) - deadline = time.monotonic() + 3 + deadline = time.monotonic() + SETTLE_SECONDS remaining = members(child.pid) while any(pid != child.pid for pid in remaining) and time.monotonic() < deadline: time.sleep(.01) @@ -121,8 +127,8 @@ def run(args, *, input, env, timeout, root): cleanup_error = exc # Signal/observation failure cannot skip an exact wait attempt. try: - child.wait(timeout=3) - deadline = time.monotonic() + 3 + child.wait(timeout=SETTLE_SECONDS) + deadline = time.monotonic() + SETTLE_SECONDS while members(child.pid) and time.monotonic() < deadline: time.sleep(.01) if members(child.pid): From 349a55716c97abe3269054b4aa0d6c9fda361f30 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Wed, 7 Oct 2026 00:09:52 +0900 Subject: [PATCH 22/22] fix(kpi): count engine block and monitor actions in SOC KPIs Suricata EVE and Coraza audit events are stored as block or monitor, so the blocked and monitored KPI counts, the Prometheus gauges and the event action filter missed them. Both spellings are now one class. A Suricata severity-1 alert is counted as blocked even when the request was not dropped. --- crates/waf-ids-core/src/lib.rs | 16 ++- docs/analytics/soc-kpis.md | 2 +- src/lib.rs | 22 ++-- tests/engine_event_kpi_visibility.rs | 153 +++++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 10 deletions(-) create mode 100644 tests/engine_event_kpi_visibility.rs diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index ff8baf67..39af963f 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -1015,6 +1015,18 @@ pub fn kpi_snapshot(data: &AppData) -> SocKpiSnapshot { kpi_snapshot_at(data, unix_now()) } +/// SOC outcome class of a recorded event action. Gateway decisions record +/// `blocked`/`monitored`; the Suricata EVE and Coraza audit adapters keep the +/// engine vocabulary `block`/`monitor`. Both spellings are one class, so KPIs, +/// Prometheus gauges and triage filters count engine events as documented. +pub fn event_action_class(action: &str) -> Option<&'static str> { + match action { + "blocked" | "block" => Some("blocked"), + "monitored" | "monitor" => Some("monitored"), + _ => None, + } +} + pub fn kpi_snapshot_at(data: &AppData, now_unix: u64) -> SocKpiSnapshot { let feed_freshness = threat_feed_freshness_snapshot(&data.threat_feeds, now_unix); SocKpiSnapshot { @@ -1028,12 +1040,12 @@ pub fn kpi_snapshot_at(data: &AppData, now_unix: u64) -> SocKpiSnapshot { blocked_event_count: data .events .iter() - .filter(|event| event.action == "blocked") + .filter(|event| event_action_class(&event.action) == Some("blocked")) .count(), monitor_event_count: data .events .iter() - .filter(|event| event.action == "monitored") + .filter(|event| event_action_class(&event.action) == Some("monitored")) .count(), audit_log_count: data.audit_logs.len(), gateway_mode: "rust-first edge gateway program baseline".to_string(), diff --git a/docs/analytics/soc-kpis.md b/docs/analytics/soc-kpis.md index b322d695..e999d953 100644 --- a/docs/analytics/soc-kpis.md +++ b/docs/analytics/soc-kpis.md @@ -37,4 +37,4 @@ ## MVP Measurement -The baseline exposes `GET /api/kpis` with counts for routes, indicators, DNSBL entries, threat feeds, fresh feeds, stale feeds, events, blocked events, monitored events, and management audit logs. `GET /api/commercial/evidence-manifest` adds the buyer-facing checklist that maps those signals to required runtime endpoints, committed documents, and deployment assets. Latency, precision, triage time, and full feed freshness percentages require the next telemetry and analyst-disposition work. +The baseline exposes `GET /api/kpis` with counts for routes, indicators, DNSBL entries, threat feeds, fresh feeds, stale feeds, events, blocked events, monitored events, and management audit logs. A blocked or monitored count covers both recorded action spellings: gateway decisions (`blocked`, `monitored`) and the Suricata EVE / Coraza audit adapters (`block`, `monitor`). A Suricata severity-1 alert is recorded as `block`, and therefore counted as blocked, even when Suricata itself let the request through; "blocked" here means the engine's block verdict, not an observed drop. `GET /api/commercial/evidence-manifest` adds the buyer-facing checklist that maps those signals to required runtime endpoints, committed documents, and deployment assets. Latency, precision, triage time, and full feed freshness percentages require the next telemetry and analyst-disposition work. diff --git a/src/lib.rs b/src/lib.rs index 532ab82e..e546174b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,8 @@ pub use waf_ids_core::{ NewAuditLogEntry, ProductEdition, ReadinessCheck, ReadinessStatus, RouteConfig, ScoredRequest, SecurityEvent, Severity, SignatureInfo, SocKpiSnapshot, TARGET_SALE_VALUE_KRW, ThreatFeedFreshness, ThreatFeedImport, ThreatFeedImportResult, ThreatFeedStatus, - ThreatIndicator, export_dnsbl_zone, ip_in_network, reverse_ipv4_for_dnsbl, score_request, + ThreatIndicator, event_action_class, export_dnsbl_zone, ip_in_network, reverse_ipv4_for_dnsbl, + score_request, }; mod coraza_audit; @@ -1047,12 +1048,19 @@ async fn list_events( ) -> Json> { let data = state.inner.read().await; let mut events: Vec = match &query.action { - Some(action) => data - .events - .iter() - .filter(|event| &event.action == action) - .cloned() - .collect(), + Some(action) => { + // A known class matches both gateway and engine spellings; an + // unknown filter keeps exact-match semantics. + let wanted = event_action_class(action); + data.events + .iter() + .filter(|event| match wanted { + Some(class) => event_action_class(&event.action) == Some(class), + None => &event.action == action, + }) + .cloned() + .collect() + } None => data.events.clone(), }; if let Some(limit) = query.limit { diff --git a/tests/engine_event_kpi_visibility.rs b/tests/engine_event_kpi_visibility.rs new file mode 100644 index 00000000..74ae6aa3 --- /dev/null +++ b/tests/engine_event_kpi_visibility.rs @@ -0,0 +1,153 @@ +//! SOC KPI visibility for proven-engine events. +//! docs/analytics/soc-kpis.md: `GET /api/kpis` counts "blocked events, monitored +//! events"; docs/architecture.md: Suricata alerts "become `SecurityEvent` rows for +//! SOC export/KPI". Gateway decisions record `blocked`/`monitored`, while the +//! Suricata EVE and Coraza audit adapters record `block`/`monitor`. Both +//! vocabularies must reach the same KPI, Prometheus and triage-filter classes. +//! Synthetic in-memory state and credentials only; no network. +use axum::{ + body::{Body, to_bytes}, + http::{Request, StatusCode}, +}; +use serde_json::Value; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, build_app}; + +const TOKEN: &str = "synthetic-kpi-token"; + +async fn send(app: &axum::Router, request: Request) -> (StatusCode, Vec) { + let response = app.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), 1_000_000).await.unwrap(); + (status, bytes.to_vec()) +} + +fn get(path: &str) -> Request { + Request::builder().uri(path).body(Body::empty()).unwrap() +} + +fn post(path: &str, token: bool, content_type: &str, body: &str) -> Request { + let mut builder = Request::builder() + .method("POST") + .uri(path) + .header("content-type", content_type); + if token { + builder = builder.header("x-admin-token", TOKEN); + } + builder.body(Body::from(body.to_string())).unwrap() +} + +async fn json(app: &axum::Router, path: &str) -> Value { + let (status, bytes) = send(app, get(path)).await; + assert_eq!(status, StatusCode::OK, "{path}"); + serde_json::from_slice(&bytes).unwrap() +} + +fn metric(text: &str, name: &str) -> u64 { + text.lines() + .find_map(|line| line.strip_prefix(&format!("{name} "))) + .unwrap_or_else(|| panic!("missing metric {name}: {text}")) + .trim() + .parse() + .unwrap() +} + +const SURICATA: &str = concat!( + r#"{"event_type":"alert","src_ip":"203.0.113.51","alert":{"signature":"ET block sev1","severity":1},"http":{"url":"/a"}}"#, + "\n", + r#"{"event_type":"alert","src_ip":"203.0.113.52","alert":{"signature":"ET monitor sev3","severity":3},"http":{"url":"/b"}}"#, + "\n" +); +const CORAZA: &str = r#"{"transaction":{"client_ip":"203.0.113.53","is_interrupted":true,"request":{"uri":"/c"},"response":{"http_code":403}},"messages":[{"message":"SQLi","data":{"id":942100,"severity":2}}]}"#; + +#[tokio::test] +async fn engine_events_are_counted_in_blocked_and_monitored_kpis() { + let app = build_app(AppState::seeded(Some(TOKEN.to_string()))); + + // Positive control: the seeded monitor route records one gateway `monitored` event. + let (status, _) = send(&app, get("/gateway/demo/probe?q=union%20select")).await; + assert_eq!(status, StatusCode::OK); + let control = json(&app, "/api/kpis").await; + assert_eq!(control["event_count"], 1, "{control}"); + assert_eq!(control["monitor_event_count"], 1, "{control}"); + assert_eq!(control["blocked_event_count"], 0, "{control}"); + + // Unauthenticated engine ingest is rejected and does not change the KPIs. + for (path, content_type, body) in [ + ("/api/ids/suricata/eve", "application/x-ndjson", SURICATA), + ("/api/waf/coraza/audit", "application/json", CORAZA), + ] { + let (status, _) = send(&app, post(path, false, content_type, body)).await; + assert_eq!(status, StatusCode::UNAUTHORIZED, "{path}"); + } + assert_eq!(json(&app, "/api/kpis").await, control); + + let (status, body) = send( + &app, + post( + "/api/ids/suricata/eve", + true, + "application/x-ndjson", + SURICATA, + ), + ) + .await; + assert_eq!( + status, + StatusCode::CREATED, + "{}", + String::from_utf8_lossy(&body) + ); + let (status, body) = send( + &app, + post("/api/waf/coraza/audit", true, "application/json", CORAZA), + ) + .await; + assert_eq!( + status, + StatusCode::CREATED, + "{}", + String::from_utf8_lossy(&body) + ); + + // The adapters keep their own vocabulary in stored events. + let events = json(&app, "/api/events").await; + let mut actions: Vec = events + .as_array() + .unwrap() + .iter() + .map(|event| event["action"].as_str().unwrap().to_string()) + .collect(); + actions.sort(); + assert_eq!(actions, ["block", "block", "monitor", "monitored"]); + + // Every recorded event belongs to exactly one KPI class. + let kpis = json(&app, "/api/kpis").await; + assert_eq!(kpis["event_count"], 4, "{kpis}"); + assert_eq!(kpis["blocked_event_count"], 2, "{kpis}"); + assert_eq!(kpis["monitor_event_count"], 2, "{kpis}"); + + let (status, bytes) = send(&app, get("/metrics")).await; + assert_eq!(status, StatusCode::OK); + let text = String::from_utf8(bytes).unwrap(); + assert_eq!(metric(&text, "waf_ids_security_events"), 4); + assert_eq!(metric(&text, "waf_ids_security_events_blocked"), 2); + assert_eq!(metric(&text, "waf_ids_security_events_monitored"), 2); + + // SOC triage filters by class in both vocabularies. + for (filter, expected) in [ + ("blocked", 2), + ("block", 2), + ("monitored", 2), + ("monitor", 2), + ] { + let filtered = json(&app, &format!("/api/events?action={filter}")).await; + assert_eq!( + filtered.as_array().unwrap().len(), + expected, + "action={filter}: {filtered}" + ); + } + let unknown = json(&app, "/api/events?action=allowed").await; + assert_eq!(unknown.as_array().unwrap().len(), 0); +}