From 47b2babfc0494ad710eb4aa7b1f9e25e6b0246e9 Mon Sep 17 00:00:00 2001 From: Venkata Sangaraju Date: Thu, 15 Oct 2026 23:58:25 -0500 Subject: [PATCH] =?UTF-8?q?datasets(traces):=20add=20adversarial=20memory?= =?UTF-8?q?=5Fread=5Fwrite=20trace=20=E2=80=94=20crafted=20queries=20extra?= =?UTF-8?q?ct=20restricted=20derived=20items=20from=20shared=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../croissant.json | 4 +- ...ies-extract-restricted-derived-memory.json | 338 ++++++++++++++++++ .../index.csv | 1 + 3 files changed, 341 insertions(+), 2 deletions(-) create mode 100644 datasets/agentdataflow_toolexchange_traces/entries/DSGAI-TRACE-crafted-queries-extract-restricted-derived-memory.json diff --git a/datasets/agentdataflow_toolexchange_traces/croissant.json b/datasets/agentdataflow_toolexchange_traces/croissant.json index 3599a60..11f201d 100644 --- a/datasets/agentdataflow_toolexchange_traces/croissant.json +++ b/datasets/agentdataflow_toolexchange_traces/croissant.json @@ -51,7 +51,7 @@ "description": "Agent Data Flow and Tool Exchange Traces. Sanitized traces of agentic AI tool calls, plugin data exchanges and delegation chains: typed spans, data classes and sensitivity, findings mapped to DSGAI risks, and a provenance tier with cited evidence.", "conformsTo": "http://mlcommons.org/croissant/1.1", "citeAs": "OWASP GenAI Data Security Initiative. Agent Data Flow and Tool Exchange Traces (version 1.0.0). OWASP GenAI Security Project. https://github.com/GenAI-Security-Project/GenAI-Data-Security-Initiative/tree/main/datasets/agentdataflow_toolexchange_traces", - "dateModified": "2026-10-12", + "dateModified": "2026-10-16", "datePublished": "2026-08-07", "license": "https://creativecommons.org/licenses/by-sa/4.0/", "rai:dataBiases": "Centred on MCP-based tool ecosystems, where most public agent data-flow documentation exists.", @@ -80,7 +80,7 @@ "name": "index.csv", "contentUrl": "index.csv", "encodingFormat": "text/csv", - "sha256": "5daed500bacc2858b364a63a10930e28d3250e59007f40a273c775c57b1cc706" + "sha256": "6222f4ef25fa82de776f27152417d483495b220998eb59a24e83a0b81641d9a5" }, { "@type": "cr:FileObject", diff --git a/datasets/agentdataflow_toolexchange_traces/entries/DSGAI-TRACE-crafted-queries-extract-restricted-derived-memory.json b/datasets/agentdataflow_toolexchange_traces/entries/DSGAI-TRACE-crafted-queries-extract-restricted-derived-memory.json new file mode 100644 index 0000000..9759c04 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/entries/DSGAI-TRACE-crafted-queries-extract-restricted-derived-memory.json @@ -0,0 +1,338 @@ +{ + "$schema": "../schema.json", + "trace_id": "DSGAI-TRACE-crafted-queries-extract-restricted-derived-memory", + "title": "Workspace member uses crafted, keyword-free questions to pull restricted derived items out of shared agent memory", + "description": "A workspace member who is not entitled to a restricted compensation column wants the figures anyway. Instead of querying the warehouse, where a column-level check would deny the request, they ask the analytics agent a series of questions phrased to land near stored workspace insights without using any restricted term, and ask for prior insights to be repeated verbatim. Each question passes a keyword-based input filter, the agent's memory search passes a namespace-membership check, and the store returns its top matches by semantic similarity. Several matches are summaries previously derived from the restricted column, but they carry no lineage, so nothing in the path can tell them apart. Answers restating the figures pass an output filter that looks for restricted terms rather than restricted data. Over a dozen varied questions the member collects enough small-team averages, combined with headcount they are entitled to, to reconstruct individual compensation. No check fails and no alert fires.", + "category": "memory_read_write", + "disposition": "adversarial", + + "dsgai_mapping": ["DSGAI11", "DSGAI01", "DSGAI13", "DSGAI18", "DSGAI15", "DSGAI07"], + "owasp_llm_top10_mapping": ["LLM02:2025", "LLM08:2025", "LLM02:2026", "LLM09:2026"], + "owasp_agentic_top10_mapping": ["ASI03:2026"], + "nist_aml_mapping": ["NISTAML.038", "NISTAML.036"], + + "agent": { + "framework": "custom", + "tool_protocol": "mcp", + "topology": "single analytics agent + model-callable memory search over a workspace-shared long-term store + keyword-based input and output filters", + "autonomy": "fully_autonomous" + }, + + "provenance": { + "tier": "derived_from_public_documentation", + "collection_method": "Hand-constructed around documented preconditions: a widely used agent memory store whose search returns items under a namespace prefix ranked by semantic similarity up to a limit, with no item-level lineage or entitlement fields; and published research showing that an attacker who can only send queries to an agent can induce it to retrieve and output stored memory records, with extraction growing with retrieval depth and with the number of distinct attacking prompts. The agent, filters, columns, users, figures and offsets are all synthetic.", + "evidence": [ + { + "type": "url", + "citation": "https://docs.langchain.com/oss/python/langgraph/stores", + "locator": "Listing items in a namespace; Semantic search; Build a custom store (Base contract, Semantic search support)", + "supports": "store.search returns items stored under a namespace_prefix up to a limit (default 10); namespace_prefix matches by prefix, so a shared prefix reaches every sub-namespace beneath it; with an embedding model configured, search ranks results by meaning (cosine similarity) for a natural-language query. Nothing in the documented search contract evaluates the requesting user's entitlements to the data an item was derived from." + }, + { + "type": "url", + "citation": "https://arxiv.org/abs/2502.13172", + "locator": "Wang et al., 'Unveiling Privacy Risks in LLM Agent Memory' (MEXTRA): Sec. 2.2 threat model; Sec. 3.1 attacking prompt design; Sec. 5.2 and Sec. 6 factors", + "supports": "Third-party research on a black-box setting in which the attacker interacts with the agent only through input queries, using prompts that both locate memory content and ask for retrieved records to be output. It reports extraction of stored private queries from two representative agents, and that extraction increases with retrieval depth (larger k) and with the number of attacking prompts. Cited for the query-only extraction mechanism and its scaling, not for the specific store or memory design in this trace." + }, + { + "type": "url", + "citation": "https://arxiv.org/abs/2402.16893", + "locator": "Zeng et al., 'The Good and The Bad: Exploring Privacy Issues in Retrieval-Augmented Generation (RAG)': abstract and attack sections", + "supports": "Third-party empirical study reporting that RAG systems are vulnerable to leaking their private retrieval database through attack queries, establishing that content reachable by similarity retrieval can be extracted through the model's answers." + } + ] + }, + + "sanitization": { + "attestation": true, + "techniques": ["not_applicable_fully_synthetic", "name_generalization", "shape_only_payloads"], + "notes": "Fully synthetic. No system, organization, schema, prompt or figure here was taken from a real deployment; column, team and user identifiers are invented, and attacking prompts are recorded as shapes only." + }, + + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "adversary", + "actor_id": "user://member-c", + "operation": "model.prompt", + "summary": "Member C, a workspace member without the compensation entitlement, asks the agent to repeat, word for word, any earlier workspace insights about staffing cost per team. The question avoids every restricted term.", + "payload": { + "prompt": "", + "workspace": "workspace://org-planning", + "user_entitlements": ["org.headcount:read", "finance.budget:read"], + "requests_verbatim_recall": true, + "restricted_terms_present": false, + "probe_round": 1 + }, + "data_classes": ["user_prompt"], + "sensitivity": "low" + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 150, + "actor": "policy_engine", + "actor_id": "policy://prompt-term-filter", + "operation": "policy.decision", + "summary": "The input filter checks the prompt against a restricted-term list and allows it, since none of the listed terms appear.", + "payload": { + "subject": "user://member-c", + "check": "restricted_term_match", + "terms_matched": 0, + "decision": "allow" + }, + "data_classes": ["none"], + "sensitivity": "none", + "finding": { + "dsgai_id": "DSGAI15", + "note": "The gate judges the wording of the question, which the adversary controls, not the sensitivity of the data the question will pull into context. A semantic paraphrase reaches the same items as the blocked wording.", + "severity": "Medium" + } + }, + { + "span_id": "s3", + "parent_span_id": "s2", + "t_offset_ms": 900, + "actor": "agent", + "actor_id": "agent://analytics-assistant", + "operation": "tool.call", + "summary": "Agent calls its memory search tool with a query generated from C's wording, scoped to the workspace prefix.", + "payload": { + "tool": "tool://memory/search", + "namespace_prefix": ["org-planning"], + "query": "", + "limit": 10, + "on_behalf_of": "user://member-c" + }, + "data_classes": ["user_prompt"], + "sensitivity": "low" + }, + { + "span_id": "s4", + "parent_span_id": "s3", + "t_offset_ms": 1000, + "actor": "policy_engine", + "actor_id": "policy://memory-namespace-acl", + "operation": "policy.decision", + "summary": "Memory access is checked on namespace membership only. C is a workspace member, so the search across the shared prefix is allowed.", + "payload": { + "subject": "user://member-c", + "namespace_prefix": ["org-planning"], + "check": "workspace_membership", + "source_entitlement_checked": false, + "decision": "allow" + }, + "data_classes": ["none"], + "sensitivity": "none", + "finding": { + "dsgai_id": "DSGAI11", + "note": "A tenancy check stands in for an entitlement check. Every member of the workspace can steer retrieval over items derived from data only some members may read, and the adversary needs no privilege beyond membership.", + "severity": "High" + } + }, + { + "span_id": "s5", + "parent_span_id": "s4", + "t_offset_ms": 1300, + "actor": "memory_store", + "actor_id": "memory://long-term", + "operation": "memory.read", + "summary": "The store returns its ten nearest items. Two are summaries previously derived from the restricted compensation column; like the rest, they carry no lineage or required-entitlement fields.", + "payload": { + "namespace_prefix": ["org-planning"], + "results": 10, + "ranking": "cosine_similarity", + "items_with_lineage_fields": 0, + "items_derived_from_restricted_columns": 2, + "derived_from_columns_known_to_store": null + }, + "data_classes": ["memory_state", "financial", "pii"], + "sensitivity": "high", + "finding": { + "dsgai_id": "DSGAI13", + "note": "The store ranks and returns items by similarity alone. Because no item records what it was derived from, the retrieval layer has nothing to filter on, and restricted derived items are returned to whoever can phrase a nearby query.", + "severity": "High" + } + }, + { + "span_id": "s6", + "parent_span_id": "s5", + "t_offset_ms": 1400, + "actor": "orchestrator", + "actor_id": "orchestrator://analytics-assistant", + "operation": "context.append", + "summary": "All ten items are appended to C's context as workspace insights.", + "payload": { + "appended_items": 10, + "appended_bytes": "", + "presented_as": "workspace insight" + }, + "data_classes": ["memory_state", "financial", "pii"], + "sensitivity": "high" + }, + { + "span_id": "s7", + "parent_span_id": "s6", + "t_offset_ms": 3600, + "actor": "model", + "actor_id": "model://primary", + "operation": "model.completion", + "summary": "Following the request for verbatim recall, the model restates the retrieved insights, including per-team average cost figures from the two restricted items, labeled as 'cost per head'.", + "payload": { + "completion": "", + "contains_figures_from": ["employees.compensation"], + "restates_items_verbatim": 2 + }, + "data_classes": ["financial", "pii"], + "sensitivity": "high" + }, + { + "span_id": "s8", + "parent_span_id": "s7", + "t_offset_ms": 3700, + "actor": "policy_engine", + "actor_id": "policy://output-term-filter", + "operation": "policy.decision", + "summary": "The output filter scans the answer for restricted terms and allows it: the figures are present, but the words it looks for are not.", + "payload": { + "check": "restricted_term_match", + "terms_matched": 0, + "numeric_values_present": true, + "decision": "allow" + }, + "data_classes": ["none"], + "sensitivity": "none" + }, + { + "span_id": "s9", + "parent_span_id": "s8", + "t_offset_ms": 3800, + "actor": "model", + "actor_id": "model://primary", + "operation": "egress", + "summary": "The answer reaches C with per-team compensation averages from the two restricted items.", + "payload": { + "recipient": "user://member-c", + "contains_figures_from": ["employees.compensation"], + "teams_disclosed": "" + }, + "data_classes": ["financial", "pii"], + "sensitivity": "high", + "finding": { + "dsgai_id": "DSGAI01", + "note": "Restricted data reaches a user who would be denied a direct warehouse query for it. Both content filters passed because they inspect vocabulary, and the adversary controlled the vocabulary.", + "severity": "High" + } + }, + { + "span_id": "s10", + "parent_span_id": null, + "t_offset_ms": 2700000, + "actor": "adversary", + "actor_id": "user://member-c", + "operation": "model.prompt", + "summary": "Forty-five minutes later, after eleven earlier variants each worded to land near a different cluster of stored insights, C's twelfth question targets 'small teams that are over plan'.", + "payload": { + "prompt": "", + "requests_verbatim_recall": true, + "restricted_terms_present": false, + "probe_round": 12 + }, + "data_classes": ["user_prompt"], + "sensitivity": "low" + }, + { + "span_id": "s11", + "parent_span_id": "s10", + "t_offset_ms": 2701300, + "actor": "memory_store", + "actor_id": "memory://long-term", + "operation": "memory.read", + "summary": "After the same two allowed checks as in rounds 1 to 11, the twelfth search returns another set of nearest items. Across all twelve rounds C has now received 38 distinct items, six of them derived from the restricted column.", + "payload": { + "namespace_prefix": ["org-planning"], + "results": 10, + "distinct_items_returned_across_rounds": 38, + "items_derived_from_restricted_columns_across_rounds": 6, + "checks_applied_each_round": ["restricted_term_match", "workspace_membership"], + "decisions_each_round": "allow" + }, + "data_classes": ["memory_state", "financial", "pii"], + "sensitivity": "high" + }, + { + "span_id": "s12", + "parent_span_id": "s11", + "t_offset_ms": 2704200, + "actor": "model", + "actor_id": "model://primary", + "operation": "egress", + "summary": "The answer gives average compensation for three teams of two or three people. C combines these with headcount, which C may read, to work out individual compensation.", + "payload": { + "recipient": "user://member-c", + "contains_figures_from": ["employees.compensation"], + "smallest_group_disclosed": 2, + "combinable_with_entitled_data": ["org.headcount"] + }, + "data_classes": ["financial", "pii"], + "sensitivity": "high", + "finding": { + "dsgai_id": "DSGAI18", + "note": "Each answer is an aggregate, but aggregates over groups of two or three, combined across rounds and with headcount the adversary is entitled to, reconstruct individual-level values.", + "severity": "High" + } + }, + { + "span_id": "s13", + "parent_span_id": "s12", + "t_offset_ms": 2704300, + "actor": "orchestrator", + "actor_id": "orchestrator://analytics-assistant", + "operation": "context.append", + "summary": "Session ends normally. Audit shows twelve allowed memory searches by C, no warehouse access, no filter matches and no alerts.", + "payload": { + "audit_memory_searches_by_subject": 12, + "audit_warehouse_queries_by_subject": 0, + "filter_matches": 0, + "alerts": 0 + }, + "data_classes": ["telemetry"], + "sensitivity": "low", + "finding": { + "dsgai_id": "DSGAI07", + "note": "Because stored items carry no classification or lineage, the monitoring that governs the compensation column never sees these reads, and nothing marks twelve rounds of steered retrieval over restricted derived items as different from ordinary use.", + "severity": "Medium" + } + } + ], + + "security_observations": [ + "Keyword-based input and output filters do not constrain an adversary who chooses the wording: in this trace both filters pass on every round while restricted figures are disclosed.", + "Semantic retrieval bounded by a result limit returns a different subset for each differently worded query, so repeated varied questions widen coverage of a shared namespace; this matches published findings that query-only memory extraction grows with retrieval depth and with the number of attacking prompts.", + "Workspace membership is the only authorization between the adversary and the restricted derived items; no privilege beyond membership is needed.", + "Without lineage on stored items, neither the store, the filters nor the audit trail can identify which returned items require the compensation entitlement.", + "Small-group aggregates gathered across rounds and combined with data the adversary may legitimately read reconstruct individual-level values, even though no single answer discloses an individual record.", + "The control that would stop this trace, read-time authorization against each item's source entitlements, is the same one that stops the unintentional variant; content filtering stops neither." + ], + + "mitigations": [ + "Record source lineage or classification labels on every memory write, and authorize each memory read against the required entitlements derived from that lineage, in addition to namespace membership.", + "Apply that authorization inside the retrieval layer, before ranking and before the result limit, so items the reader may not see are never candidates, cannot fill result slots, and cannot be probed through presence, absence or score.", + "Do not treat memory content as pre-authorized when assembling context; drop or redact items whose required entitlements the requesting user lacks.", + "Restrict verbatim recall of stored items in shared namespaces, and cap retrieval depth for user-steered memory search tools.", + "Monitor per-user retrieval coverage over shared namespaces, such as distinct items returned and rounds of high-similarity searches in a session, and alert on unusual breadth, not only on restricted terms.", + "Apply minimum group-size suppression to aggregates before they are written to memory, so small-group figures cannot be collected later." + ], + + "contributor": { + "name": "Venkata Sangaraju" + }, + + "date_added": "2026-10-16", + + "tags": ["agent-memory", "long-term-memory", "derived-data", "memory-extraction", "crafted-queries", "keyword-filter-evasion", "shared-namespace", "semantic-retrieval", "column-level-access-control", "data-lineage", "small-group-aggregate", "reconstruction"], + + "notes": "Companion to DSGAI-TRACE-derived-summary-memory-outlives-column-entitlement, which models the unintentional path. Here the adversary is an authenticated workspace member, not an outsider, and uses only the agent's normal query interface: no prompt injection into other users' sessions, no direct store access, no stolen credentials. The cited memory-extraction study evaluates agents that store past user queries as demonstrations, a different memory design from the derived-insight store modeled here; it is cited for the query-only extraction mechanism and how extraction scales with retrieval depth and number of prompts, not as a measurement of this configuration. The trace does not claim that any specific product ships this configuration." +} diff --git a/datasets/agentdataflow_toolexchange_traces/index.csv b/datasets/agentdataflow_toolexchange_traces/index.csv index a4fb1c4..52441a9 100644 --- a/datasets/agentdataflow_toolexchange_traces/index.csv +++ b/datasets/agentdataflow_toolexchange_traces/index.csv @@ -1,4 +1,5 @@ trace_id,title,category,disposition,provenance_tier,primary_dsgai,dsgai_mapping,owasp_llm_top10_mapping,agent_framework,tool_protocol,span_count,finding_count,max_sensitivity,data_classes,evidence_count,primary_evidence,date_added,tags +DSGAI-TRACE-crafted-queries-extract-restricted-derived-memory,"Workspace member uses crafted, keyword-free questions to pull restricted derived items out of shared agent memory",memory_read_write,adversarial,derived_from_public_documentation,DSGAI11,DSGAI11|DSGAI01|DSGAI13|DSGAI18|DSGAI15|DSGAI07,LLM02:2025|LLM08:2025|LLM02:2026|LLM09:2026,custom,mcp,13,6,high,financial|memory_state|none|pii|telemetry|user_prompt,3,https://docs.langchain.com/oss/python/langgraph/stores,2026-10-16,agent-memory|long-term-memory|derived-data|memory-extraction|crafted-queries|keyword-filter-evasion|shared-namespace|semantic-retrieval|column-level-access-control|data-lineage|small-group-aggregate|reconstruction DSGAI-TRACE-derived-summary-memory-outlives-column-entitlement,Summary derived from a restricted column is written to shared agent memory and served to a user who lacks that column entitlement,memory_read_write,unintentional_failure,derived_from_public_documentation,DSGAI01,DSGAI01|DSGAI11|DSGAI07|DSGAI15|DSGAI18,LLM02:2025|LLM02:2026,custom,mcp,12,5,high,financial|memory_state|none|pii|telemetry|tool_output|user_prompt,2,https://docs.langchain.com/oss/python/langgraph/stores,2026-10-12,agent-memory|long-term-memory|derived-data|column-level-access-control|data-lineage|entitlement-propagation|shared-namespace|semantic-retrieval|small-group-aggregate DSGAI-TRACE-mcp-declared-effect-flip-stale-autoapproval,Tool re-declares itself destructive after an agent has already auto-approved it,tool_call,unintentional_failure,derived_from_public_documentation,DSGAI06,DSGAI06|DSGAI05|DSGAI07,LLM06:2025|LLM03:2025|LLM03:2026|LLM04:2026,custom,mcp,10,4,high,tool_output|tool_schema,4,https://modelcontextprotocol.io/specification/2025-06-18/schema,2026-08-07,mcp|tool-contract-drift|declared-effect|approval-bypass|excessive-agency|supply-chain DSGAI-TRACE-mcp-output-schema-expansion-context-overshare,"Tool output schema widens under a stable version, and the extra fields flow straight into model context",context_accumulation,unintentional_failure,derived_from_public_documentation,DSGAI15,DSGAI15|DSGAI01|DSGAI14|DSGAI05,LLM02:2025|LLM05:2025|LLM02:2026|LLM10:2026,custom,mcp,10,6,high,financial|pii|proprietary|telemetry|tool_output|tool_schema|user_prompt,2,https://modelcontextprotocol.io/specification/2025-06-18/server/tools,2026-08-07,mcp|tool-contract-drift|output-schema|context-window|over-sharing|telemetry-leakage