diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d2065d..a8df53f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). scope changes (LLM01 cross-modal 2.2.3/2.2.4, LLM04 artifact provenance 3.1.1/3.1.3, LLM05 fine-tuning subversion 6.1.2/3.5.1, LLM08 hidden context 10.2.4, LLM10 generated code 9.3.7). AISVS has no requirement for scanning generated code itself; the LLM10 section says so. +- **Optional `verification_method` mapping field** (#101). Records how an assessor can check the control is implemented. + Non-breaking: rows without it are unchanged. Documented in `docs/SCHEMA_V2_MIGRATION.md`; not exported to OLIR. + Piloted in `agentic-top10/Agentic_AIUC1.md` (all values DRAFT pending SME review), with methods adapted from the + NPW Agentic AI Control Catalogue. Next: npm publish to npmjs.com, custom domain (crosswalk.owasp.org), vendor integration packs, NeMo Guardrails configs. diff --git a/agentic-top10/Agentic_AIUC1.md b/agentic-top10/Agentic_AIUC1.md index d3414e7..1b5b95b 100644 --- a/agentic-top10/Agentic_AIUC1.md +++ b/agentic-top10/Agentic_AIUC1.md @@ -2,9 +2,11 @@ OWASP GenAI Crosswalk Source list : OWASP Top 10 for Agentic Applications 2026 (ASI01–ASI10) Framework : AIUC-1 — The standard for AI agent security, safety and reliability - Version : 2026-Q1 + Version : 2026-Q4 Maintained by: OWASP GenAI Data Security Initiative — https://genai.owasp.org License : CC BY-SA 4.0 + Verification: Methods adapted from the NPW Agentic AI Control Catalogue v2.4.0, + New Pacific Way Ltd., CC BY-SA 4.0 — https://www.newpacificway.com/ai-controls --> # Agentic Top 10 2026 × AIUC-1 @@ -109,12 +111,12 @@ silent data exfiltration engine via indirect prompt injection. #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Third-party testing of adversarial robustness | B001 | Mandatory adversarial testing program validating resilience against prompt injection and goal manipulation attempts | Foundational | Both | -| Detect adversarial input | B002 | Runtime monitoring to detect and respond to adversarial inputs redirecting agent goals | Hardening | Both | -| Implement real-time input filtering | B005 | Automated moderation filtering inputs before they reach agent reasoning | Foundational | Both | -| Prevent unauthorized AI agent actions | B006 | Safeguards preventing agents from performing actions beyond authorised scope — directly blocks goal hijack execution | Foundational | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Third-party testing of adversarial robustness | B001 | Mandatory adversarial testing program validating resilience against prompt injection and goal manipulation attempts | Foundational | Both | DRAFT — Injection test executed through each distinct ingestion path + Inject a sub-goal in test; confirm detection and that the attempt is counted (NPW C02, C07); evidence: current third-party test report | +| Detect adversarial input | B002 | Runtime monitoring to detect and respond to adversarial inputs redirecting agent goals | Hardening | Both | DRAFT — Inject a sub-goal in test; confirm detection and that the attempt is counted + Introduce a known deviation and confirm the auditing model flags it (NPW C07, H08) | +| Implement real-time input filtering | B005 | Automated moderation filtering inputs before they reach agent reasoning | Foundational | Both | DRAFT — Filter evaluated against a defined test set + Run a maintained payload test corpus (NPW C05, C03) | +| Prevent unauthorized AI agent actions | B006 | Safeguards preventing agents from performing actions beyond authorised scope — directly blocks goal hijack execution | Foundational | Both | DRAFT — Agent Charter present, current, and matching observed behaviour + Attempt objective modification through the reflection path; confirm rejection (NPW B01, C08) | #### Mitigations by tier @@ -167,12 +169,12 @@ attacker. #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Third-party testing of adversarial robustness | B001 | Adversarial testing specifically covering tool misuse and unsafe tool chaining | Foundational | Both | -| Prevent AI endpoint scraping | B004 | Safeguards preventing probing of tool endpoints and rate-limit abuse | Foundational | Both | -| Prevent unauthorized AI agent actions | B006 | Core control — agents restricted to narrowest permission set per tool, blocking misuse by design | Foundational | Both | -| Enforce user access privileges to AI systems | B007 | User-level access controls ensuring agents cannot invoke tools beyond what the authorising user can access | Foundational | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Third-party testing of adversarial robustness | B001 | Adversarial testing specifically covering tool misuse and unsafe tool chaining | Foundational | Both | DRAFT — Execute a known policy-circumventing chain in a test environment + Attempt an out-of-scope tool call and confirm denial (NPW H09, A08); evidence: current third-party test report | +| Prevent AI endpoint scraping | B004 | Safeguards preventing probing of tool endpoints and rate-limit abuse | Foundational | Both | DRAFT — Review limit configuration; force a breach in a test environment (NPW B04) | +| Prevent unauthorized AI agent actions | B006 | Core control — agents restricted to narrowest permission set per tool, blocking misuse by design | Foundational | Both | DRAFT — Diff the tool registry against the per-agent allowlist + Inspect grant and revocation records for a sampled task (NPW B02, B09) | +| Enforce user access privileges to AI systems | B007 | User-level access controls ensuring agents cannot invoke tools beyond what the authorising user can access | Foundational | Both | DRAFT — Attempt an out-of-scope tool call and confirm denial (NPW A08); repeat for several users with different permissions | #### Mitigations by tier @@ -224,12 +226,12 @@ enabling attacker reuse across sessions and environments. #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Data & Privacy (full domain) | A | All agent credential and identity data falls under Domain A — PII protection, access restrictions, data leakage prevention | Foundational | Both | -| Enforce user access privileges to AI systems | B007 | Agents operate only within the privilege envelope of the authorising user — no escalation permitted | Foundational | Both | -| Protect model deployment environment | B008 | Secure deployment including encryption, access controls, and authorisation for the agent's runtime environment | Foundational | Both | -| Accountability (full domain) | E | Audit trails and logging of all agent identity use — essential for detecting privilege abuse post-incident | Hardening | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Data & Privacy (full domain) | A | All agent credential and identity data falls under Domain A — PII protection, access restrictions, data leakage prevention | Foundational | Both | | +| Enforce user access privileges to AI systems | B007 | Agents operate only within the privilege envelope of the authorising user — no escalation permitted | Foundational | Both | DRAFT — Trace a downstream system call and confirm the principal is resolvable + Escalation attempt in a test environment (NPW A04, F03) | +| Protect model deployment environment | B008 | Secure deployment including encryption, access controls, and authorisation for the agent's runtime environment | Foundational | Both | DRAFT — Secret scanning across agent runtime and logs + Inspect credential store; confirm maximum time-to-live against policy (NPW E08, A02) | +| Accountability (full domain) | E | Audit trails and logging of all agent identity use — essential for detecting privilege abuse post-incident | Hardening | Both | | #### Mitigations by tier @@ -282,12 +284,12 @@ behaviour. #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Third-party testing of adversarial robustness | B001 | Supply chain components included in adversarial test scope | Foundational | Both | -| Manage public release of technical details | B003 | Controls preventing over-disclosure of agent architecture details that enable supply chain targeting | Hardening | Both | -| Protect model deployment environment | B008 | Secure deployment environment including verification of all runtime components | Foundational | Both | -| Data & Privacy (full domain) | A | All data shared with supply chain components governed under Domain A | Foundational | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Third-party testing of adversarial robustness | B001 | Supply chain components included in adversarial test scope | Foundational | Both | DRAFT — Confirm tools, connectors and MCP servers are in the scope of the current third-party adversarial test report | +| Manage public release of technical details | B003 | Controls preventing over-disclosure of agent architecture details that enable supply chain targeting | Hardening | Both | | +| Protect model deployment environment | B008 | Secure deployment environment including verification of all runtime components | Foundational | Both | DRAFT — Verification record for each deployed component + Verify signatures; reconcile the SBOM against deployed components (NPW E02, E09) | +| Data & Privacy (full domain) | A | All data shared with supply chain components governed under Domain A | Foundational | Both | | #### Mitigations by tier @@ -337,12 +339,12 @@ the agent's code generation capability. #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Third-party testing of adversarial robustness | B001 | Code generation and execution paths explicitly included in adversarial test scope | Foundational | Both | -| Implement real-time input filtering | B005 | Filter inputs before they reach code generation components | Foundational | Build | -| Prevent unauthorized AI agent actions | B006 | Agents cannot execute code outside defined, sandboxed execution environments | Foundational | Build | -| Limit output over-exposure | B009 | Code outputs filtered and validated before execution | Foundational | Build | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Third-party testing of adversarial robustness | B001 | Code generation and execution paths explicitly included in adversarial test scope | Foundational | Both | DRAFT — Sandbox escape test + Submit code with a known finding through the pipeline; confirm it is blocked (NPW B06, B14); evidence: current third-party test report | +| Implement real-time input filtering | B005 | Filter inputs before they reach code generation components | Foundational | Build | | +| Prevent unauthorized AI agent actions | B006 | Agents cannot execute code outside defined, sandboxed execution environments | Foundational | Build | DRAFT — Sandbox escape test + Confirm sandbox lifecycle configuration; verify no state survives an execution (NPW B06, B10) | +| Limit output over-exposure | B009 | Code outputs filtered and validated before execution | Foundational | Build | DRAFT — Submit code with a known finding through the pipeline; confirm it is blocked (NPW B14) | #### Mitigations by tier @@ -393,11 +395,11 @@ malicious data in persistent memory, enabling long-term behavioural manipulation #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Data & Privacy (full domain) | A | Agent memory and context treated as sensitive data — subject to all Domain A controls | Foundational | Both | -| Detect adversarial input | B002 | Monitor memory write operations for adversarial patterns | Hardening | Both | -| Implement real-time input filtering | B005 | Filter content before it is committed to persistent memory stores | Hardening | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Data & Privacy (full domain) | A | Agent memory and context treated as sensitive data — subject to all Domain A controls | Foundational | Both | | +| Detect adversarial input | B002 | Monitor memory write operations for adversarial patterns | Hardening | Both | DRAFT — Inject a known anomaly and confirm the alert fires + Log sample showing write source for each memory entry (NPW D06, D04) | +| Implement real-time input filtering | B005 | Filter content before it is committed to persistent memory stores | Hardening | Both | DRAFT — Attempt to commit an unverified fact and confirm the gate fires (NPW D07) | #### Mitigations by tier @@ -446,11 +448,11 @@ multi-agent clusters. #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Enforce user access privileges to AI systems | B007 | Agent-to-agent trust boundaries enforced with same rigour as user-to-agent boundaries | Foundational | Build | -| Protect model deployment environment | B008 | Secure deployment including encrypted inter-agent communication channels | Foundational | Build | -| Accountability (full domain) | E | Full audit trail of all inter-agent messages — essential for incident reconstruction | Hardening | Build | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Enforce user access privileges to AI systems | B007 | Agent-to-agent trust boundaries enforced with same rigour as user-to-agent boundaries | Foundational | Build | DRAFT — Attempt a cross-zone instruction from a lower to a higher trust zone + Escalation attempt in a test environment (NPW F09, F03) | +| Protect model deployment environment | B008 | Secure deployment including encrypted inter-agent communication channels | Foundational | Build | DRAFT — Attempt to spoof an agent message and confirm rejection (NPW F01) | +| Accountability (full domain) | E | Full audit trail of all inter-agent messages — essential for incident reconstruction | Hardening | Build | | #### Mitigations by tier @@ -500,11 +502,11 @@ into physical process control. See ISA/IEC 62443 and NIST SP 800-82 crosswalks f #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Reliability (full domain) | D | Availability, consistency, failure recovery, and rate limiting — the primary domain for cascading failure prevention | Foundational | Both | -| Prevent unauthorized AI agent actions | B006 | Blast radius limitation — agents cannot take actions beyond scope even during failure propagation | Foundational | Both | -| Accountability (full domain) | E | Incident logging and traceability — essential for identifying cascade origin and containment | Hardening | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Reliability (full domain) | D | Availability, consistency, failure recovery, and rate limiting — the primary domain for cascading failure prevention | Foundational | Both | | +| Prevent unauthorized AI agent actions | B006 | Blast radius limitation — agents cannot take actions beyond scope even during failure propagation | Foundational | Both | DRAFT — Review limit configuration; force a breach in a test environment + Attempt a high-risk operation with a single agent and confirm it is blocked (NPW B04, F08) | +| Accountability (full domain) | E | Incident logging and traceability — essential for identifying cascade origin and containment | Hardening | Both | | #### Mitigations by tier @@ -552,12 +554,12 @@ performing actions that appear legitimate in audit logs while being agent-driven #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Safety (full domain) | C | Harm prevention and human oversight — the primary AIUC-1 domain for trust exploitation | Foundational | Both | -| Society (full domain) | F | Bias, transparency, and societal impact — covers manipulation and deceptive agent behaviour | Foundational | Both | -| Limit output over-exposure | B009 | Output filtering preventing agents from producing manipulative or deceptive content | Foundational | Both | -| Accountability (full domain) | E | Audit trail distinguishing agent-driven actions from genuinely human-initiated actions | Hardening | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Safety (full domain) | C | Harm prevention and human oversight — the primary AIUC-1 domain for trust exploitation | Foundational | Both | | +| Society (full domain) | F | Bias, transparency, and societal impact — covers manipulation and deceptive agent behaviour | Foundational | Both | | +| Limit output over-exposure | B009 | Output filtering preventing agents from producing manipulative or deceptive content | Foundational | Both | | +| Accountability (full domain) | E | Audit trail distinguishing agent-driven actions from genuinely human-initiated actions | Hardening | Both | | #### Mitigations by tier @@ -597,13 +599,13 @@ or ASI06 attack that has gone undetected. #### AIUC-1 mapping -| Control | ID | Description | Tier | Scope | -|---|---|---|---|---| -| Third-party testing of adversarial robustness | B001 | Rogue agent detection included in adversarial test scope | Foundational | Both | -| Detect adversarial input | B002 | Runtime behavioural monitoring detecting deviation from intended agent purpose | Hardening | Both | -| Prevent unauthorized AI agent actions | B006 | Scope constraints — rogue agent cannot act outside defined boundaries even if internal goals are compromised | Foundational | Both | -| Safety (full domain) | C | Safety guardrails as the last line of defence against rogue agent outputs | Foundational | Both | -| Accountability (full domain) | E | Full audit trail enabling rogue agent detection, attribution, and forensics | Foundational | Both | +| Control | ID | Description | Tier | Scope | Verification method | +|---|---|---|---|---|---| +| Third-party testing of adversarial robustness | B001 | Rogue agent detection included in adversarial test scope | Foundational | Both | DRAFT — Induce a deviation and confirm the alert + Discovery run record and disposition of findings (NPW H05, H07); evidence: current third-party test report | +| Detect adversarial input | B002 | Runtime behavioural monitoring detecting deviation from intended agent purpose | Hardening | Both | DRAFT — Induce a deviation and confirm the alert (NPW H05) | +| Prevent unauthorized AI agent actions | B006 | Scope constraints — rogue agent cannot act outside defined boundaries even if internal goals are compromised | Foundational | Both | | +| Safety (full domain) | C | Safety guardrails as the last line of defence against rogue agent outputs | Foundational | Both | | +| Accountability (full domain) | E | Full audit trail enabling rogue agent detection, attribution, and forensics | Foundational | Both | | #### Mitigations by tier @@ -682,6 +684,7 @@ See: | Date | Version | Change | Author | |---|---|---|---| | 2026-03-24 | 2026-Q1 | Initial mapping — all 10 ASI entries | OWASP GenAI Data Security Initiative | +| 2026-10-01 | 2026-Q4 | Pilot: Verification method column (#101). 22 of 26 requirement-level rows filled. All values DRAFT pending SME review. Methods adapted from NPW Agentic AI Control Catalogue; text after the NPW citation is contributor-authored. | a-moskvin | --- diff --git a/data/entries/ASI01.json b/data/entries/ASI01.json index 7c7da36..a9c2902 100644 --- a/data/entries/ASI01.json +++ b/data/entries/ASI01.json @@ -777,6 +777,7 @@ "control_name": "Third-party testing of adversarial robustness", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Injection test executed through each distinct ingestion path + Inject a sub-goal in test; confirm detection and that the attempt is counted (NPW C02, C07); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -786,6 +787,7 @@ "control_name": "Detect adversarial input", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Inject a sub-goal in test; confirm detection and that the attempt is counted + Introduce a known deviation and confirm the auditing model flags it (NPW C07, H08)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -795,6 +797,7 @@ "control_name": "Implement real-time input filtering", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Filter evaluated against a defined test set + Run a maintained payload test corpus (NPW C05, C03)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -804,6 +807,7 @@ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Agent Charter present, current, and matching observed behaviour + Attempt objective modification through the reflection path; confirm rejection (NPW B01, C08)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI02.json b/data/entries/ASI02.json index 585136b..797ec3c 100644 --- a/data/entries/ASI02.json +++ b/data/entries/ASI02.json @@ -771,6 +771,7 @@ "control_name": "Third-party testing of adversarial robustness", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Execute a known policy-circumventing chain in a test environment + Attempt an out-of-scope tool call and confirm denial (NPW H09, A08); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -780,6 +781,7 @@ "control_name": "Prevent AI endpoint scraping", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Review limit configuration; force a breach in a test environment (NPW B04)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -789,6 +791,7 @@ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Diff the tool registry against the per-agent allowlist + Inspect grant and revocation records for a sampled task (NPW B02, B09)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -798,6 +801,7 @@ "control_name": "Enforce user access privileges to AI systems", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Attempt an out-of-scope tool call and confirm denial (NPW A08); repeat for several users with different permissions", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI03.json b/data/entries/ASI03.json index 62088fd..d3e42ea 100644 --- a/data/entries/ASI03.json +++ b/data/entries/ASI03.json @@ -766,6 +766,7 @@ "control_name": "Enforce user access privileges to AI systems", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Trace a downstream system call and confirm the principal is resolvable + Escalation attempt in a test environment (NPW A04, F03)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -775,6 +776,7 @@ "control_name": "Protect model deployment environment", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Secret scanning across agent runtime and logs + Inspect credential store; confirm maximum time-to-live against policy (NPW E08, A02)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI04.json b/data/entries/ASI04.json index 82ef2d3..70f2a02 100644 --- a/data/entries/ASI04.json +++ b/data/entries/ASI04.json @@ -757,6 +757,7 @@ "control_name": "Third-party testing of adversarial robustness", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Confirm tools, connectors and MCP servers are in the scope of the current third-party adversarial test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -775,6 +776,7 @@ "control_name": "Protect model deployment environment", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Verification record for each deployed component + Verify signatures; reconcile the SBOM against deployed components (NPW E02, E09)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI05.json b/data/entries/ASI05.json index 03f548d..4eb0eaf 100644 --- a/data/entries/ASI05.json +++ b/data/entries/ASI05.json @@ -747,6 +747,7 @@ "control_name": "Third-party testing of adversarial robustness", "tier": "Foundational", "scope": "Build", + "verification_method": "DRAFT — Sandbox escape test + Submit code with a known finding through the pipeline; confirm it is blocked (NPW B06, B14); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -765,6 +766,7 @@ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Build", + "verification_method": "DRAFT — Sandbox escape test + Confirm sandbox lifecycle configuration; verify no state survives an execution (NPW B06, B10)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -774,6 +776,7 @@ "control_name": "Limit output over-exposure", "tier": "Foundational", "scope": "Build", + "verification_method": "DRAFT — Submit code with a known finding through the pipeline; confirm it is blocked (NPW B14)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI06.json b/data/entries/ASI06.json index f0265d8..9ba7efb 100644 --- a/data/entries/ASI06.json +++ b/data/entries/ASI06.json @@ -755,6 +755,7 @@ "control_name": "Detect adversarial input", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Inject a known anomaly and confirm the alert fires + Log sample showing write source for each memory entry (NPW D06, D04)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -764,6 +765,7 @@ "control_name": "Implement real-time input filtering", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Attempt to commit an unverified fact and confirm the gate fires (NPW D07)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI07.json b/data/entries/ASI07.json index c656a22..bee766a 100644 --- a/data/entries/ASI07.json +++ b/data/entries/ASI07.json @@ -746,6 +746,7 @@ "control_name": "Enforce user access privileges to AI systems", "tier": "Hardening", "scope": "Build", + "verification_method": "DRAFT — Attempt a cross-zone instruction from a lower to a higher trust zone + Escalation attempt in a test environment (NPW F09, F03)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -755,6 +756,7 @@ "control_name": "Protect model deployment environment", "tier": "Hardening", "scope": "Build", + "verification_method": "DRAFT — Attempt to spoof an agent message and confirm rejection (NPW F01)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI08.json b/data/entries/ASI08.json index 79d2ed8..29cc3e9 100644 --- a/data/entries/ASI08.json +++ b/data/entries/ASI08.json @@ -755,6 +755,7 @@ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Review limit configuration; force a breach in a test environment + Attempt a high-risk operation with a single agent and confirm it is blocked (NPW B04, F08)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/entries/ASI10.json b/data/entries/ASI10.json index 055645b..54dfa85 100644 --- a/data/entries/ASI10.json +++ b/data/entries/ASI10.json @@ -763,6 +763,7 @@ "control_name": "Third-party testing of adversarial robustness", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Induce a deviation and confirm the alert + Discovery run record and disposition of findings (NPW H05, H07); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -772,6 +773,7 @@ "control_name": "Detect adversarial input", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Induce a deviation and confirm the alert (NPW H05)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/data/schema.json b/data/schema.json index 63f7840..cc97bb8 100644 --- a/data/schema.json +++ b/data/schema.json @@ -164,6 +164,10 @@ "type": "string", "description": "The framework release this row was authored against, e.g. \"ISO 27001:2022\". Pinning it is what makes version rot detectable instead of silent." }, + "verification_method": { + "type": "string", + "description": "How can we check that the control is implemented for this risk: the evidence, test or artefact to examine. Optional. Not exported to OLIR, which has no corresponding field." + }, "reviewed_by": { "type": "array", "items": { diff --git a/docs/SCHEMA_V2_MIGRATION.md b/docs/SCHEMA_V2_MIGRATION.md index ce4dd7f..55818d0 100644 --- a/docs/SCHEMA_V2_MIGRATION.md +++ b/docs/SCHEMA_V2_MIGRATION.md @@ -36,6 +36,7 @@ than a rewrite of it. | `framework_version` | string | crosswalk | | `reviewed_by` | string[] | crosswalk | | `review_date` | date | crosswalk | +| `verification_method` | string | crosswalk | ### `relationship` — the logical comparison @@ -103,6 +104,19 @@ against ASVS 4.0.3 and a mapping written against 5.0 are indistinguishable. `reviewed_by` holds named humans. An agent never adds itself. +### `verification_method` — how the control is checked + +`verification_method` says how an assessor confirms the control is actually in +place: evidence, validation test(s), or artefacts to inspect. + +The field is optional and non-breaking. Rows without it are unchanged. + +- Method is a security judgment, so it lands as `DRAFT — ` pending SME + review, the same as `relationship` and `confidence`. +- Where a method is adapted from an external source, cite the specific control + in the cell and attribute the source in the file header. +- It is not exported to OLIR, which has no corresponding field. + --- ## Migration state diff --git a/docs/data.js b/docs/data.js index e529169..c6134f4 100644 --- a/docs/data.js +++ b/docs/data.js @@ -13696,6 +13696,7 @@ window.CROSSWALK_DATA = [ "control_name": "Third-party testing of adversarial robustness", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Injection test executed through each distinct ingestion path + Inject a sub-goal in test; confirm detection and that the attempt is counted (NPW C02, C07); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -13705,6 +13706,7 @@ window.CROSSWALK_DATA = [ "control_name": "Detect adversarial input", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Inject a sub-goal in test; confirm detection and that the attempt is counted + Introduce a known deviation and confirm the auditing model flags it (NPW C07, H08)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -13714,6 +13716,7 @@ window.CROSSWALK_DATA = [ "control_name": "Implement real-time input filtering", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Filter evaluated against a defined test set + Run a maintained payload test corpus (NPW C05, C03)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -13723,6 +13726,7 @@ window.CROSSWALK_DATA = [ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Agent Charter present, current, and matching observed behaviour + Attempt objective modification through the reflection path; confirm rejection (NPW B01, C08)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -15052,6 +15056,7 @@ window.CROSSWALK_DATA = [ "control_name": "Third-party testing of adversarial robustness", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Execute a known policy-circumventing chain in a test environment + Attempt an out-of-scope tool call and confirm denial (NPW H09, A08); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -15061,6 +15066,7 @@ window.CROSSWALK_DATA = [ "control_name": "Prevent AI endpoint scraping", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Review limit configuration; force a breach in a test environment (NPW B04)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -15070,6 +15076,7 @@ window.CROSSWALK_DATA = [ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Diff the tool registry against the per-agent allowlist + Inspect grant and revocation records for a sampled task (NPW B02, B09)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -15079,6 +15086,7 @@ window.CROSSWALK_DATA = [ "control_name": "Enforce user access privileges to AI systems", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Attempt an out-of-scope tool call and confirm denial (NPW A08); repeat for several users with different permissions", "confidence": "unreviewed", "reviewed_by": [] }, @@ -16358,6 +16366,7 @@ window.CROSSWALK_DATA = [ "control_name": "Enforce user access privileges to AI systems", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Trace a downstream system call and confirm the principal is resolvable + Escalation attempt in a test environment (NPW A04, F03)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -16367,6 +16376,7 @@ window.CROSSWALK_DATA = [ "control_name": "Protect model deployment environment", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Secret scanning across agent runtime and logs + Inspect credential store; confirm maximum time-to-live against policy (NPW E08, A02)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -17705,6 +17715,7 @@ window.CROSSWALK_DATA = [ "control_name": "Third-party testing of adversarial robustness", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Confirm tools, connectors and MCP servers are in the scope of the current third-party adversarial test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -17723,6 +17734,7 @@ window.CROSSWALK_DATA = [ "control_name": "Protect model deployment environment", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Verification record for each deployed component + Verify signatures; reconcile the SBOM against deployed components (NPW E02, E09)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -18903,6 +18915,7 @@ window.CROSSWALK_DATA = [ "control_name": "Third-party testing of adversarial robustness", "tier": "Foundational", "scope": "Build", + "verification_method": "DRAFT — Sandbox escape test + Submit code with a known finding through the pipeline; confirm it is blocked (NPW B06, B14); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -18921,6 +18934,7 @@ window.CROSSWALK_DATA = [ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Build", + "verification_method": "DRAFT — Sandbox escape test + Confirm sandbox lifecycle configuration; verify no state survives an execution (NPW B06, B10)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -18930,6 +18944,7 @@ window.CROSSWALK_DATA = [ "control_name": "Limit output over-exposure", "tier": "Foundational", "scope": "Build", + "verification_method": "DRAFT — Submit code with a known finding through the pipeline; confirm it is blocked (NPW B14)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -20114,6 +20129,7 @@ window.CROSSWALK_DATA = [ "control_name": "Detect adversarial input", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Inject a known anomaly and confirm the alert fires + Log sample showing write source for each memory entry (NPW D06, D04)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -20123,6 +20139,7 @@ window.CROSSWALK_DATA = [ "control_name": "Implement real-time input filtering", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Attempt to commit an unverified fact and confirm the gate fires (NPW D07)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -21286,6 +21303,7 @@ window.CROSSWALK_DATA = [ "control_name": "Enforce user access privileges to AI systems", "tier": "Hardening", "scope": "Build", + "verification_method": "DRAFT — Attempt a cross-zone instruction from a lower to a higher trust zone + Escalation attempt in a test environment (NPW F09, F03)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -21295,6 +21313,7 @@ window.CROSSWALK_DATA = [ "control_name": "Protect model deployment environment", "tier": "Hardening", "scope": "Build", + "verification_method": "DRAFT — Attempt to spoof an agent message and confirm rejection (NPW F01)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -22436,6 +22455,7 @@ window.CROSSWALK_DATA = [ "control_name": "Prevent unauthorized AI agent actions", "tier": "Foundational", "scope": "Both", + "verification_method": "DRAFT — Review limit configuration; force a breach in a test environment + Attempt a high-risk operation with a single agent and confirm it is blocked (NPW B04, F08)", "confidence": "unreviewed", "reviewed_by": [] }, @@ -24731,6 +24751,7 @@ window.CROSSWALK_DATA = [ "control_name": "Third-party testing of adversarial robustness", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Induce a deviation and confirm the alert + Discovery run record and disposition of findings (NPW H05, H07); evidence: current third-party test report", "confidence": "unreviewed", "reviewed_by": [] }, @@ -24740,6 +24761,7 @@ window.CROSSWALK_DATA = [ "control_name": "Detect adversarial input", "tier": "Hardening", "scope": "Both", + "verification_method": "DRAFT — Induce a deviation and confirm the alert (NPW H05)", "confidence": "unreviewed", "reviewed_by": [] }, diff --git a/scripts/generate.js b/scripts/generate.js index 8bd5858..913dc13 100644 --- a/scripts/generate.js +++ b/scripts/generate.js @@ -423,6 +423,7 @@ function applyV2(mapping, cells) { if (rt) mapping.rationale_type = rt; if (cells.rationale) mapping.rationale = cells.rationale.replace(/\s+/g, ' ').trim(); if (cells.framework_version) mapping.framework_version = cells.framework_version.trim(); + if (cells.verification_method) mapping.verification_method = cells.verification_method.replace(/\s+/g, ' ').trim(); if (/^(true|yes|broad)$/i.test((cells.broad_applicability || '').trim())) { mapping.broad_applicability = true; } @@ -453,6 +454,7 @@ function v2HeaderIndex(tableText) { const idx = { relationship: -1, rationale_type: -1, rationale: -1, confidence: -1, framework_version: -1, reviewed_by: -1, review_date: -1, broad_applicability: -1, + verification_method: -1, }; const headerLine = tableText.split('\n').find((l) => l.trim().startsWith('|')); if (!headerLine) return idx; @@ -467,6 +469,7 @@ function v2HeaderIndex(tableText) { else if (/^reviewed by|^reviewer/.test(h)) idx.reviewed_by = i; else if (/^review date/.test(h)) idx.review_date = i; else if (/^broad/.test(h)) idx.broad_applicability = i; + else if (/^verification method/.test(h)) idx.verification_method = i; }); return idx; } @@ -617,6 +620,7 @@ function parseControlTable(sectionBody, frameworkName, qr) { reviewed_by: header.reviewed_by >= 0 ? cols[header.reviewed_by] : '', review_date: header.review_date >= 0 ? cols[header.review_date] : '', broad_applicability: header.broad_applicability >= 0 ? cols[header.broad_applicability] : '', + verification_method: header.verification_method >= 0 ? cols[header.verification_method] : '', }); mappings.push(mapping); diff --git a/scripts/generate.test.mjs b/scripts/generate.test.mjs index 16417d0..d029a00 100644 --- a/scripts/generate.test.mjs +++ b/scripts/generate.test.mjs @@ -98,6 +98,43 @@ test('no mapping claims a confidence without a named reviewer', () => { assert.deepEqual(offenders.slice(0, 5), [], `${offenders.length} unreviewed row(s) claim confidence`); }); +test('Verification method columns are parsed, not filed as notes', () => { + // Collect every cell under a "Verification method" header in the sources. + // A header is a table row followed by a |---| separator row. + const cells = new Set(); + const split = (l) => l.split('|').slice(1, -1).map((x) => x.trim()); + for (const dir of ['llm-top10', 'agentic-top10', 'dsgai-2026', 'ast-top10']) { + for (const f of fs.readdirSync(path.join(ROOT, dir)).filter((n) => n.endsWith('.md'))) { + const lines = fs.readFileSync(path.join(ROOT, dir, f), 'utf8').split(/\r?\n/); + let col = -1; + lines.forEach((line, n) => { + if (!line.trim().startsWith('|')) { col = -1; return; } + const next = lines[n + 1] || ''; + if (/^\s*\|[\s:|-]+\|\s*$/.test(next)) { + col = split(line).findIndex((x) => /^verification method/i.test(x)); + return; + } + if (/^\s*\|[\s:|-]+\|\s*$/.test(line)) return; + const c = split(line); + if (col >= 0 && c[col]) cells.add(c[col].replace(/\s+/g, ' ')); + }); + } + } + if (!cells.size) return; // no file carries the column yet + + let parsed = 0; + const leaked = []; + for (const f of fs.readdirSync(ENTRIES).filter((n) => n.endsWith('.json'))) { + const e = JSON.parse(fs.readFileSync(path.join(ENTRIES, f), 'utf8')); + for (const m of e.mappings || []) { + if (m.verification_method) parsed++; + if (m.notes && cells.has(m.notes)) leaked.push(`${e.id}/${m.framework}:${m.control_id}`); + } + } + assert.ok(parsed > 0, 'a source file has a Verification method column but no mapping carries verification_method'); + assert.deepEqual(leaked.slice(0, 5), [], `${leaked.length} row(s) stored the verification method as notes`); +}); + test('DRAFT never survives into a stored enum field', () => { // The Markdown templates carry the literal word DRAFT in the relationship, // rationale and confidence columns. generate.js is supposed to resolve those diff --git a/src/index.ts b/src/index.ts index 84989cb..2b0f8fd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,8 @@ export interface Mapping { control_name: string; tier?: string; notes?: string; + /** How to verify the control is implemented for this risk. Optional. */ + verification_method?: string; /** * Distinct incidents that exemplify this entry and record a CONFIRMED failure * of this control. Generated; present only on rows with linked failures.