From fe14166d0c73464c688b3614e01cce434fcbb07b Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:02:06 -0400 Subject: [PATCH 01/10] feat(dice-storage): Drivine drift-report store and observed-schema source Three scoped drift-report reads that scope before limiting, ordered by capture instant with a persisted per-schema sequence tie-break so pages are repeatable; injective scope keys (global vs ctx:). Observed schema identifies dice bookkeeping by property shape rather than name, so same-named domain types stay visible and the metamodel never observes its own nodes as drift. End-to-end drift-check IT: declare, ingest an undeclared mention, live run, report persisted, proposition quarantined. Refs #45; stacks on feat/metamodel-drift. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- CHANGELOG.md | 27 + .../dice/storage/DrivineDriftReportStore.kt | 240 +++++++++ .../storage/DrivineObservedSchemaSource.kt | 249 +++++++++ .../dice/storage/MetamodelRowMappers.kt | 119 ++++- .../embabel/dice/storage/MetamodelSchema.kt | 65 +++ .../DrivineDriftCheckIntegrationTest.kt | 229 ++++++++ .../DrivineDriftReportStoreIntegrationTest.kt | 505 ++++++++++++++++++ ...vineObservedSchemaSourceIntegrationTest.kt | 369 +++++++++++++ .../embabel/dice/storage/TestApplication.kt | 38 +- docs/design/architecture.md | 2 +- docs/design/metamodel-drift.md | 53 +- 11 files changed, 1873 insertions(+), 23 deletions(-) create mode 100644 dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt create mode 100644 dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt create mode 100644 dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index da5ca334..dc475d5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -345,6 +345,33 @@ and the consumer PRs that deliver it). graph stores. Aliases arrive in this same Unreleased block, so no consumer can be in that state on a published build. The API is additive: three read-only accessors and one public constant, and no existing signature changed. +- Drivine/Neo4j-backed drift persistence in `dice-storage`. `DrivineDriftReportStore` keeps each + check as a `(:MetamodelDriftReport)` node, MERGEd on the natural key + `(schemaName, versionHash, capturedAt, contextKey)` — `contextKey` being `global` or `ctx:`, + since a Cypher MERGE cannot key on a null; prefixing every real context keeps the encoding + injective, so no `ContextId` value can share a key with the global bucket and silently rewrite its + scope. All three bounded reads are separate statements that push their scope + into the query *before* the `LIMIT`, which is what stops a schema whose recent history is mostly + context-scoped from reporting zero global drift while plenty sits in the store. Ordering is newest + first by capture instant, stored as `(epochSecond, nano)` so both the sort and an inclusive `since` + window stay exact below the millisecond, with a per-schema `(:MetamodelDriftReportCounter)` + sequence breaking exact ties so a limited page is repeatable. `DrivineObservedSchemaSource` takes + the snapshot: `db.labels()`/`db.relationshipTypes()` unscoped, and per context the distinct mention + types plus the edges whose `sourcePropositions` name that context's propositions. The unscoped + path subtracts dice's own bookkeeping — every proposition, provenance, lineage, collector-trace + and metamodel node label, and the `HAS_MENTION`/`DERIVED_FROM`/`SCORED`/`RETIRED_IN` edges — so + governance never observes the nodes its own last run wrote as domain drift. That subtraction is by + *shape*, not by name: a label is hidden only while every node carrying it matches dice's shape for + it, and an edge type only while none of its edges carries `sourcePropositions`, so an app + governing a type genuinely called `Source` still sees it reported. `MetamodelSchema` + collects the uniqueness constraints these stores need alongside the label list the observer + excludes, so the two cannot drift apart. Still no Spring wiring; that arrives in the autoconfigure + slice. + **Compatibility: additive.** New classes in an existing module; no existing API touched. Hosts + that already declared the three `MetamodelVersion`/`MetamodelSchemaCounter` constraints by hand can + swap in `MetamodelSchema.specs()`, which is a superset — the drift-report store needs three more: + `MetamodelDriftReport(schemaName, versionHash, capturedAt, contextKey)`, + `MetamodelDriftReportCounter(schemaName)`, and `MetamodelDriftReport(schemaName, sequence)`. - Optional source revisions in the `dice` core provenance model, the first slice of DICE #64. `ProvenanceEntry` gains a sixth field, `sourceRevision`: an opaque, provider-defined string, non-blank when present, recording which version of a source a claim was read from. diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt new file mode 100644 index 00000000..87ed04ea --- /dev/null +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt @@ -0,0 +1,240 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import com.embabel.agent.core.ContextId +import com.embabel.dice.metamodel.DriftReport +import com.embabel.dice.metamodel.DriftReportStore +import org.drivine.manager.PersistenceManager +import org.drivine.query.QuerySpecification +import org.slf4j.LoggerFactory +import org.springframework.transaction.annotation.Transactional +import java.time.Instant + +/** + * Drivine / Neo4j implementation of [DriftReportStore]: keeps every drift check as a + * `(:MetamodelDriftReport)` node. + * + * The write MERGEs on the natural key `(schemaName, versionHash, capturedAt, contextKey)`, so a + * retry writes the same node again rather than a second copy of one observation. That is only + * race-free under a uniqueness constraint on the same four properties — see [MetamodelSchema], and + * see [DriftReportRowMapper.GLOBAL_CONTEXT_KEY] for why the fourth is `contextKey` and not + * `contextId`. + * + * Every statement is parameterized; nothing caller-derived is ever interpolated into Cypher. + * + * ## Scope goes into the query, never into a filter afterwards + * + * The contract's hardest rule, and the one its test pins: each of the three reads has its own + * statement, with its scope in the `WHERE` clause and the `LIMIT` applied after it. The tempting + * shortcut — read one limited page and filter it down in Kotlin — applies the limit *before* the + * scope, so a schema whose recent history happens to be mostly context-scoped would report zero + * global drift while plenty sat in the store. That is a wrong answer that looks exactly like a + * right one, which is why there are three statements here rather than one and a `filter`. + * + * ## Newest first means the capture instant, broken by a counter + * + * The order is [DriftReport.capturedAt] descending — the instant the graph was *looked at*, which + * is what the contract promises and what a `since` window bounds. Write order can't stand in for + * it: a check of last week's snapshot saved today is still last week's observation. + * + * The instant alone is not a total order, though. Two reports of one schema — a global sweep and a + * per-context one, say — can share a capture instant, and then a plain `ORDER BY` leaves their + * relative order to the database. With a `LIMIT` on top that is not merely untidy: the page + * boundary lands somewhere arbitrary, so the same read can return different rows each time and a + * caller walking the history can miss one entirely. So each report also takes the next value off a + * per-schema `(:MetamodelDriftReportCounter)` node when — and only when — its node is first + * created, and that sequence breaks ties. Same mechanism as + * [DrivineMetamodelVersionStore]'s, deliberately on its own counter node: stamps and reports have + * very different volumes (a stamp per schema change, a report per scheduled check), a shared + * counter would put every drift check in the same run into a write conflict with the version stamp + * that precedes it, and the version store's own sequence would grow gaps that mean nothing. + * + * A re-save of a report that already exists neither bumps the counter nor reassigns the sequence, + * so an idempotent write stays idempotent and a corrected observation keeps its original place. + * + * @param persistenceManager Drivine's handle on the `neo` datasource. + */ +open class DrivineDriftReportStore( + private val persistenceManager: PersistenceManager, +) : DriftReportStore { + + private val logger = LoggerFactory.getLogger(DrivineDriftReportStore::class.java) + + private companion object { + + /** + * Upsert the report node and, if this is the first time we've seen it, give it the next + * number off its schema's counter. One statement, so one transaction: a report node never + * exists without its place in the tie-break order. + * + * `WITH n WHERE n.sequence IS NULL` separates the two halves. On a re-save that filters the + * row away, so the counter is never bumped and the existing sequence is never reassigned — + * the drifted type sets are refreshed and the report keeps the position it has always had. + * + * `SET c.lockedBy = ...` writes a property nobody reads, to take the exclusive lock on the + * counter before the increment below reads it; on its own the increment is a + * read-modify-write and two concurrent savers could both read 5 and both write 6. It is + * cheap insurance rather than a proven necessity — the same measurement in + * [DrivineMetamodelVersionStore] could not tell the locked and unlocked versions apart. + * What is load-bearing is the uniqueness constraint on `(schemaName, sequence)`: a lost + * update becomes a loud, retryable constraint violation instead of a silently arbitrary + * order. + */ + private val SAVE_REPORT = """ + MERGE (n:MetamodelDriftReport { + schemaName: ${'$'}schemaName, + versionHash: ${'$'}versionHash, + capturedAt: ${'$'}capturedAt, + contextKey: ${'$'}contextKey + }) + SET n.driftedEntityTypes = ${'$'}driftedEntityTypes, + n.driftedRelationshipTypes = ${'$'}driftedRelationshipTypes, + n.capturedAtEpochSecond = ${'$'}capturedAtEpochSecond, + n.capturedAtNano = ${'$'}capturedAtNano, + n.contextId = ${'$'}contextId + WITH n + WHERE n.sequence IS NULL + MERGE (c:MetamodelDriftReportCounter {schemaName: ${'$'}schemaName}) + SET c.lockedBy = ${'$'}capturedAt + WITH n, c + SET c.sequence = coalesce(c.sequence, 0) + 1 + WITH n, c + SET n.sequence = c.sequence + """.trimIndent() + + /** + * Every read starts here. + * + * `capturedAtEpochSecond IS NOT NULL` is not defensive noise. Neo4j sorts null as the + * *largest* value, so a node missing the sort key would sort to the front of a DESC order, + * consume a slot of the caller's `limit`, and then be dropped by the mapper — hiding a + * perfectly good report behind a broken one. A node with no sort key never took a place in + * the order at all, so it is excluded in the database instead. + */ + private val MATCH_SCHEMA = """ + MATCH (n:MetamodelDriftReport {schemaName: ${'$'}schemaName}) + WHERE n.capturedAtEpochSecond IS NOT NULL + """.trimIndent() + + /** Only unscoped, whole-graph checks: a global report has no `contextId` property at all. */ + private const val ONLY_GLOBAL = "AND n.contextId IS NULL" + + /** Only one context's checks. Global reports and other contexts' are both excluded. */ + private const val ONLY_CONTEXT = "AND n.contextId = \$contextId" + + /** + * The `since` bound, inclusive, compared second-then-nanosecond so it is exact. + * + * Comparing a single truncated millisecond value would be simpler and subtly wrong: a bound + * falling part-way through a millisecond would sweep in reports captured just before it. + */ + private val SINCE_BOUND = """ + AND (n.capturedAtEpochSecond > ${'$'}sinceEpochSecond + OR (n.capturedAtEpochSecond = ${'$'}sinceEpochSecond AND n.capturedAtNano >= ${'$'}sinceNano)) + """.trimIndent() + + /** Newest first by capture instant, with the write sequence breaking exact ties. */ + private val NEWEST_FIRST_PAGE = """ + RETURN n + ORDER BY n.capturedAtEpochSecond DESC, n.capturedAtNano DESC, coalesce(n.sequence, -1) DESC + LIMIT ${'$'}limit + """.trimIndent() + } + + @Transactional + override fun saveDriftReport(report: DriftReport) { + logger.debug( + "Saving drift report schemaName={} versionHash={} contextId={} capturedAt={}", + report.schemaName, + report.versionHash.take(8), + report.contextId?.value, + report.capturedAt, + ) + persistenceManager.execute( + QuerySpecification.withStatement(SAVE_REPORT).bind(DriftReportRowMapper.bindMap(report)), + ) + } + + @Transactional(readOnly = true) + override fun driftReports(schemaName: String, limit: Int, since: Instant?): List = + readPage(scope = null, schemaName = schemaName, limit = limit, since = since) + + @Transactional(readOnly = true) + override fun globalDriftReports(schemaName: String, limit: Int, since: Instant?): List = + readPage(scope = ONLY_GLOBAL, schemaName = schemaName, limit = limit, since = since) + + @Transactional(readOnly = true) + override fun driftReportsInContext( + schemaName: String, + contextId: ContextId, + limit: Int, + since: Instant?, + ): List = readPage( + scope = ONLY_CONTEXT, + schemaName = schemaName, + limit = limit, + since = since, + extraBindings = mapOf("contextId" to contextId.value), + ) + + /** + * Assemble and run one of the three scoped reads. + * + * The statement is built from the constants above and nothing else — `scope` is one of this + * class's own literals, never anything a caller supplied — so this stays string *assembly*, not + * string interpolation of user data. Every value still travels as a bound parameter. + * + * A corrupt row that survives the query is warned about and skipped rather than taking down the + * whole governance read; see [DriftReportRowMapper], which throws instead of inventing defaults + * precisely so this can happen. Note the honest consequence of a bounded read: a skipped row has + * already spent one of the caller's `limit` slots, so a page can come back shorter than asked + * for. Silently reading further to backfill would break the bound the contract exists to keep. + */ + private fun readPage( + scope: String?, + schemaName: String, + limit: Int, + since: Instant?, + extraBindings: Map = emptyMap(), + ): List { + require(limit > 0) { "limit must be positive, but was $limit" } + + val statement = buildString { + append(MATCH_SCHEMA) + scope?.let { append("\n").append(it) } + if (since != null) append("\n").append(SINCE_BOUND) + append("\n").append(NEWEST_FIRST_PAGE) + } + val bindings = buildMap { + put("schemaName", schemaName) + put("limit", limit) + putAll(extraBindings) + if (since != null) { + put("sinceEpochSecond", since.epochSecond) + put("sinceNano", since.nano) + } + } + + @Suppress("UNCHECKED_CAST") + val spec = QuerySpecification.withStatement(statement).bind(bindings) as QuerySpecification + return persistenceManager.query(spec).filterIsInstance>().mapNotNull { row -> + runCatching { DriftReportRowMapper.fromRow(row) } + .onFailure { logger.warn("Skipping unreadable MetamodelDriftReport row: {}", it.message) } + .getOrNull() + } + } +} diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt new file mode 100644 index 00000000..bfb89ef7 --- /dev/null +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt @@ -0,0 +1,249 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import com.embabel.agent.core.ContextId +import com.embabel.dice.metamodel.ObservedSchema +import com.embabel.dice.metamodel.ObservedSchemaSource +import org.drivine.manager.PersistenceManager +import org.drivine.query.QuerySpecification +import java.time.Clock + +/** + * Every node label dice writes for its own bookkeeping, and the properties that identify a node as + * really being dice's rather than merely wearing the same label. + * + * A drift check compares the *domain* schema an app declared against what a live graph actually + * holds. None of these labels was ever part of anybody's declared domain schema, so counting them + * would flag dice's own storage machinery as drift on every run — and, worse, would make governance + * report *itself*, since stamping a version and writing a report both add labels to the very graph + * the next check observes. A check that fires because the last check ran is noise that never + * settles. + * + * **The shape is here because a name is not a reservation.** Excluding the *name* `Source` would + * hide a domain type genuinely called `Source` — an undeclared one could then never appear in an + * unscoped report, which is the failure a drift check exists to prevent, made permanent and silent. + * So a label is only excluded when the nodes carrying it actually look like dice's: dice's `Source` + * nodes carry `key`, its governance nodes carry `schemaName`, and so on. A same-named node that + * doesn't match keeps the label in the observation. + * + * Each shape is the label's declared uniqueness key, extended with properties the node fragment + * writes unconditionally where those add discrimination. Deliberately kept *minimal*: a shape that + * demanded a property dice doesn't always write would make dice's own nodes look foreign and + * reintroduce the self-reporting bug, which is the worse direction to be wrong in. + * + * `DiceBookkeepingShapeTest` pins that every label in [CollectorTraceSchema.LABELS] and + * [MetamodelSchema.LABELS] has an entry here, so adding a node label to either store can't quietly + * skip this map. + */ +val DICE_BOOKKEEPING_LABEL_SHAPES: Map> = mapOf( + // Core persistence: propositions and their mentions, provenance, chunk history. + "Proposition" to listOf("id", "contextId", "text"), + "Mention" to listOf("id", "span", "type", "role"), + "Source" to listOf("key"), + "ProcessedChunk" to listOf("id"), + + // Lineage records. + "ProjectionRecord" to listOf("propositionId", "runId", "target"), + "CollectorRecord" to listOf("propositionId", "runId"), + "CollectorRun" to listOf("runId"), + + // Collector trace. Uniqueness keys only — the trace store writes these through several + // statements, and a stricter shape would risk calling its own half-written run foreign. + "CollectorTraceRun" to listOf("runId"), + "CollectorCandidateEdge" to listOf("id"), + "CollectorSignalScore" to listOf("id"), + "CollectorComponent" to listOf("id"), + "CollectorDecision" to listOf("id"), + "CollectorRetired" to listOf("id"), + + // Metamodel governance. + "MetamodelVersion" to listOf("schemaName", "contentHash"), + "MetamodelSchemaCounter" to listOf("schemaName"), + "MetamodelDriftReport" to listOf("schemaName", "versionHash", "capturedAt", "contextKey"), + "MetamodelDriftReportCounter" to listOf("schemaName"), +) + +/** The bookkeeping label names, for callers that only need the names. */ +val DICE_BOOKKEEPING_LABELS: Set = DICE_BOOKKEEPING_LABEL_SHAPES.keys + +/** + * Relationship types dice writes for its own bookkeeping, on the same grounds as + * [DICE_BOOKKEEPING_LABEL_SHAPES] and subject to the same name-is-not-a-reservation rule. + * + * Worth stating plainly, because the obvious first version of this class had no such set: dice's + * bookkeeping is *not* all node labels. `HAS_MENTION` and `DERIVED_FROM` sit on every proposition + * ever stored, so without this a whole-graph observation reports them as undeclared relationship + * drift on the very first run against a populated graph, forever. + * + * These have no per-type shape because the discriminator is the same for all of them and belongs to + * the *other* side: every edge the graph writer projects from domain data carries + * `sourcePropositions`, and no dice bookkeeping edge does. That is the positive marker the + * context-scoped path already selects on, so the whole-graph path uses it too and the two agree. + */ +val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( + "HAS_MENTION", + "DERIVED_FROM", + "SCORED", + "RETIRED_IN", +) + +/** + * Drivine / Neo4j implementation of [ObservedSchemaSource]: asks a live graph what it actually + * contains, so a `DeclaredObservedDiffer` can hold it up against what was declared. + * + * Two genuinely different observation paths, because the database offers no single query that + * answers both: + * + * - **Whole graph** (`contextId == null`) introspects the database's own catalogue — `db.labels()` + * and `db.relationshipTypes()` — and subtracts dice's bookkeeping from both sides. The + * subtraction is by *shape*, not by name: see [DICE_BOOKKEEPING_LABEL_SHAPES]. A bookkeeping name + * the domain is also using stays in the observation, so an undeclared type can still be reported. + * - **One context** (`contextId != null`) cannot use those procedures at all: they have no notion + * of a context and would answer for the whole database. It derives both sides from that context's + * own data instead: + * - entity types are the distinct `Mention.type` values on that context's propositions; + * - relationship types come from the `sourcePropositions` property the graph writer stamps on + * every edge it persists — the ids of the propositions that produced it. An edge belongs to a + * context's set when at least one of those ids names a proposition in that context. That is a + * join on a property, not a tag on the edge, and it deliberately does not collapse: an edge + * sourced from two contexts appears in both. An undeclared relationship type present in your + * context's data is drift in your context regardless of who else also produced it. + * + * The scoped entity side is deliberately *not* filtered at all. Those are Neo4j labels; a mention's + * `type` is a domain type name an extractor produced, so the two live in different namespaces and + * subtracting one from the other would only ever hide real drift from an app that happens to govern + * a type called `Source`. + * + * **Two honest limits, neither resolvable from names and shape alone.** First, exclusion is decided + * per *label*, not per node: if any node wearing a bookkeeping label fails dice's shape, the whole + * label stays observed, dice's own nodes included. That direction is chosen on purpose — a + * spuriously reported type is visible and dismissable, a silently hidden one is neither — but it + * does mean a graph mixing a domain `Source` with dice's own will report `Source` every run until + * the domain type is declared. Second, deciding this costs a scan of dice's own labels on every + * unscoped observation (each probe stops at the first non-conforming node, so it is cheap only when + * one exists). Context-scoped checks never pay it, and on a very large graph they are the ones to + * schedule. + * + * @param persistenceManager Drivine's handle on the `neo` datasource. + * @param clock Supplies the snapshot's capture instant. Injectable because that instant ends up in + * a drift report's natural key, so a test that needs two checks to be one observation — or two — + * has to be able to choose it. + */ +open class DrivineObservedSchemaSource( + private val persistenceManager: PersistenceManager, + private val clock: Clock = Clock.systemUTC(), +) : ObservedSchemaSource { + + private companion object { + + private const val ALL_LABELS = "CALL db.labels() YIELD label RETURN label" + + private const val ALL_RELATIONSHIP_TYPES = + "CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType" + + /** + * Bookkeeping labels the domain has also claimed — those carrying at least one node that + * does *not* match dice's shape for them. Whatever this returns stays in the observation. + * + * One branch per label, each stopping at the first non-conforming node, unioned into a + * single round trip. Label and property names are this file's own compile-time constants; + * nothing caller-derived is assembled in. + */ + private val LABELS_CLAIMED_BY_DOMAIN: String = DICE_BOOKKEEPING_LABEL_SHAPES.entries + .joinToString("\nUNION ALL\n") { (label, shape) -> + val notDiceShaped = shape.joinToString(" OR ") { property -> "n.$property IS NULL" } + "MATCH (n:$label) WHERE $notDiceShaped RETURN '$label' AS label LIMIT 1" + } + + /** + * Bookkeeping relationship types the domain has also claimed — those carrying at least one + * edge with `sourcePropositions`, which is the property the graph writer stamps on every + * edge it projects from domain data and no dice bookkeeping edge has. + */ + private val RELATIONSHIP_TYPES_CLAIMED_BY_DOMAIN: String = """ + MATCH ()-[r:${DICE_BOOKKEEPING_RELATIONSHIP_TYPES.joinToString("|")}]->() + WHERE r.sourcePropositions IS NOT NULL + RETURN DISTINCT type(r) + """.trimIndent() + + /** + * Entity types in one context: what its propositions actually mention. + * + * `m.type IS NOT NULL` because a null would come back as a missing element rather than a + * type name and land in the observed set as nothing useful. + */ + private val MENTION_TYPES_IN_CONTEXT = """ + MATCH (:Proposition {contextId: ${'$'}contextId})-[:HAS_MENTION]->(m:Mention) + WHERE m.type IS NOT NULL + RETURN DISTINCT m.type + """.trimIndent() + + /** + * Relationship types in one context: every edge at least one of whose source propositions + * lives there. + * + * dice's own edges have no `sourcePropositions` property, so `any(... IN null ...)` is null + * for them and they drop out here rather than needing a name-based exclusion afterwards. + * This is the shape test the whole-graph path mirrors. + */ + private val RELATIONSHIP_TYPES_IN_CONTEXT = """ + MATCH (p:Proposition {contextId: ${'$'}contextId}) + WITH collect(p.id) AS ids + MATCH ()-[r]->() + WHERE any(pid IN r.sourcePropositions WHERE pid IN ids) + RETURN DISTINCT type(r) + """.trimIndent() + } + + override fun observe(contextId: ContextId?): ObservedSchema = + if (contextId == null) observeWholeGraph() else observeContext(contextId) + + private fun observeWholeGraph(): ObservedSchema { + // Subtract only the bookkeeping the domain has *not* also claimed. A name dice uses and the + // domain also uses is the domain's for observation purposes, because failing to report an + // undeclared type is the one failure a drift check cannot recover from. + val hiddenLabels = DICE_BOOKKEEPING_LABELS - queryStrings(LABELS_CLAIMED_BY_DOMAIN) + val hiddenRelationshipTypes = + DICE_BOOKKEEPING_RELATIONSHIP_TYPES - queryStrings(RELATIONSHIP_TYPES_CLAIMED_BY_DOMAIN) + return ObservedSchema( + entityTypeNames = queryStrings(ALL_LABELS) - hiddenLabels, + relationshipTypeNames = queryStrings(ALL_RELATIONSHIP_TYPES) - hiddenRelationshipTypes, + capturedAt = clock.instant(), + ) + } + + private fun observeContext(contextId: ContextId): ObservedSchema { + val bindings = mapOf("contextId" to contextId.value) + // Neither side subtracts anything. The scoped queries are already shape-based: mention types + // are domain names by construction, and the relationship query selects on + // `sourcePropositions`, which only a projected domain edge carries. Subtracting names on top + // would be the bug this class exists to avoid — it would drop a domain relationship type + // that happens to be spelled `DERIVED_FROM` even though the query proved it was the + // domain's. + return ObservedSchema( + entityTypeNames = queryStrings(MENTION_TYPES_IN_CONTEXT, bindings), + relationshipTypeNames = queryStrings(RELATIONSHIP_TYPES_IN_CONTEXT, bindings), + capturedAt = clock.instant(), + ) + } + + /** Run a single-column query and collect its non-null values. */ + private fun queryStrings(statement: String, bindings: Map = emptyMap()): Set { + val spec = QuerySpecification.withStatement(statement).bind(bindings).transform(String::class.java) + return persistenceManager.query(spec).filterNotNull().toSet() + } +} diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt index 001e6de9..997c6152 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt @@ -16,6 +16,8 @@ package com.embabel.dice.storage import com.embabel.agent.core.Cardinality +import com.embabel.agent.core.ContextId +import com.embabel.dice.metamodel.DriftReport import com.embabel.dice.metamodel.MetamodelVersion import com.embabel.dice.metamodel.PropertySignature import com.fasterxml.jackson.core.type.TypeReference @@ -102,7 +104,7 @@ object MetamodelVersionRowMapper { entityTypeLabels = deserializeMapOfLabelSets(row.str("entityTypeLabels")), entityTypeProperties = deserializeMapOfSignatureSets(row.str("entityTypeProperties")), relationshipNames = deserializeList(row.str("relationshipNames")), - entityTypeAliases = deserializeAliasMap(row.optionalStr("entityTypeAliases")), + entityTypeAliases = deserializeAliasMap(row.strOrNull("entityTypeAliases")), ) require(version.contentHash == storedHash) { "MetamodelVersion '${version.schemaName}' fails its integrity check: stored contentHash " + @@ -112,6 +114,106 @@ object MetamodelVersionRowMapper { } } +/** + * Translate drift reports to and from the property maps the Neo4j graph store reads and writes. + * + * Neo4j properties are scalars and flat arrays, while a report's drifted type sets are collections, + * so both sets are serialized to JSON strings. JSON also handles names containing pipes, tabs, + * newlines and quotes, which these names routinely do: they come out of LLM extraction. Both sets + * are written sorted, so re-saving one observation writes byte-identical JSON and the MERGE is a + * no-op. Nothing reads the order back; the sets are read into a `Set`. + * + * The capture instant is written three ways. `capturedAt` is the ISO-8601 string and is half the + * natural key, so it is what round trips. `capturedAtEpochSecond` and `capturedAtNano` let the + * database sort and range-filter on the instant at full precision. Epoch milliseconds, which a + * version stamp uses, truncate: two reports captured 500µs apart would compare equal, leaving + * "newest first" arbitrary between them, and a `since` bound falling inside a millisecond would + * sweep in reports captured just before it. Sorting on the ISO string has its own failure — + * `Instant.toString()` writes no fraction on a whole second and `'Z'` outranks `'.'`, so + * `12:00:00Z` sorts after `12:00:00.500Z`. + * + * A fourth property, `contextKey`, encodes the report's scope; see [GLOBAL_CONTEXT_KEY]. + * + * Reads are strict: a property this mapper wrote must be present when it is read again. `contextId` + * is the one exception, because its absence is how a global report is encoded. A node missing + * anything else is corrupt, so the accessor throws and the store's surrounding guard skips the row + * with a warning. + */ +object DriftReportRowMapper { + + /** + * The stand-in `contextKey` a global (unscoped) report carries. + * + * A nullable `contextId` can't go into a MERGE key directly: Cypher property-map equality + * against a literal `null` matches nothing, including a node that has no such property, so a + * global report would take the CREATE branch on every retry and duplicate. `contextKey` is + * never null, so MERGE can key on it and a global report is as idempotent as a scoped one. + */ + const val GLOBAL_CONTEXT_KEY: String = "global" + + /** The prefix every real context's key carries. */ + private const val CONTEXT_KEY_PREFIX = "ctx:" + + /** + * The MERGE key a report's scope contributes: `global`, or `ctx:` followed by the context id. + * + * The prefix is what makes the encoding injective. `ContextId` accepts any non-blank string, so + * encoding scope as the bare context id with a sentinel standing in for global lets a caller + * name a context after the sentinel. A global report and that context's report would then share + * a MERGE key, land on one node, and each save would rewrite the other's scope, surfacing a + * context-scoped finding as whole-graph drift or the reverse. With the prefix the two spaces are + * disjoint: `global` carries no `ctx:` prefix, so no context id can produce it, and + * `ContextId("global")` maps to `ctx:global`. + */ + fun contextKeyFor(contextId: ContextId?): String = + contextId?.let { CONTEXT_KEY_PREFIX + it.value } ?: GLOBAL_CONTEXT_KEY + + /** + * Bind values for a write. The natural key is + * `(schemaName, versionHash, capturedAt, contextKey)`. + */ + fun bindMap(report: DriftReport): Map = mapOf( + "schemaName" to report.schemaName, + "versionHash" to report.versionHash, + "capturedAt" to report.capturedAt.toString(), + "capturedAtEpochSecond" to report.capturedAt.epochSecond, + "capturedAtNano" to report.capturedAt.nano, + "driftedEntityTypes" to serializeList(report.driftedEntityTypes.sorted()), + "driftedRelationshipTypes" to serializeList(report.driftedRelationshipTypes.sorted()), + "contextKey" to contextKeyFor(report.contextId), + // Null for a global report, which leaves the node with no `contextId` property at all. The + // store's global read matches on that absence. + "contextId" to report.contextId?.value, + ) + + /** + * Rebuild a [DriftReport] from a returned node's property map, checking on the way that the two + * halves of its scope still agree. + * + * The scope is stored twice: as `contextId`, absent when global, and as the never-null + * `contextKey` the natural key merges on. Reads and writes use different halves — a scoped read + * matches `contextId`, a save merges on `contextKey`. A node where the two disagree would answer + * to one scope when read and another when re-saved, so the row is refused. + */ + fun fromRow(row: Map<*, *>): DriftReport { + val contextId = row.strOrNull("contextId")?.let { ContextId(it) } + val storedKey = row.str("contextKey") + require(storedKey == contextKeyFor(contextId)) { + "MetamodelDriftReport for '${row.strOrNull("schemaName")}' fails its scope check: it is stored " + + "under contextKey '$storedKey' but reads back as " + + "${contextId?.let { "context '${it.value}'" } ?: "a global report"}" + } + return DriftReport( + schemaName = row.str("schemaName"), + versionHash = row.str("versionHash"), + driftedEntityTypes = deserializeSet(row.str("driftedEntityTypes")), + driftedRelationshipTypes = deserializeSet(row.str("driftedRelationshipTypes")), + capturedAt = Instant.parse(row.str("capturedAt")), + contextId = contextId, + ) + } +} + // Serialization helpers: JSON, for escape-safe round-trip encoding. private fun serializeList(items: List): String = @@ -123,6 +225,14 @@ private fun deserializeList(serialized: String): List = objectMapper.typeFactory.constructCollectionType(List::class.java, String::class.java) ) +/** Deserialize a JSON string back to a set — what a drift report's drifted type collections are. */ +private fun deserializeSet(serialized: String): Set = + if (serialized.isEmpty()) emptySet() + else objectMapper.readValue( + serialized, + objectMapper.typeFactory.constructCollectionType(Set::class.java, String::class.java) + ) + /** * Serialize the per-type label sets as `{"Person": ["Agent", "Entity"], ...}`. * @@ -270,7 +380,8 @@ private fun Map<*, *>.str(key: String): String = requireNotNull(this[key]) { "required property '$key' is missing from the stored node" }.toString() /** - * Read a property that may legitimately not be there, where absent means the stamp declared nothing - * to put in it. Only the alias map is read this way; everything else goes through [str]. + * Read a property whose absence is itself meaningful, rather than a fault: a stamp that declared + * no aliases, or a drift report whose check covered the whole graph and so wrote no `contextId`. + * Everything else goes through [str]. */ -private fun Map<*, *>.optionalStr(key: String): String? = this[key]?.toString() +private fun Map<*, *>.strOrNull(key: String): String? = this[key]?.toString() diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt new file mode 100644 index 00000000..b32ad411 --- /dev/null +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import org.drivine.schema.SchemaItemSpec +import org.drivine.schema.UniquenessConstraintSpec + +/** + * The constraints and node labels the metamodel governance stores need, as plain data. + * + * Two things depend on this list and they must not disagree. A host (and the integration-test + * harness) declares [specs] so the stores' MERGEs are race-free; [LABELS] is what + * [DrivineObservedSchemaSource] subtracts from an observation so governance never reports its own + * bookkeeping as domain drift. Keeping both here means adding a governance node label is one edit, + * not two edits in two modules that quietly drift apart. + */ +object MetamodelSchema { + + /** + * Every MERGE these stores perform needs its key to be unique, because a MERGE is only + * race-free when it is. Without that, concurrent saves all miss the match, all create, and the + * history fills with duplicates. + * + * The two `sequence` constraints are the safety net under the ordering. They make two records + * of one schema sharing a position impossible to store, so a lost counter update fails loudly + * and retryably instead of quietly making "newest first" arbitrary. + */ + fun specs(): List = listOf( + // Version stamps -- see DrivineMetamodelVersionStore. + UniquenessConstraintSpec(label = "MetamodelVersion", properties = listOf("schemaName", "contentHash")), + UniquenessConstraintSpec(label = "MetamodelSchemaCounter", property = "schemaName"), + UniquenessConstraintSpec(label = "MetamodelVersion", properties = listOf("schemaName", "sequence")), + + // Drift reports -- see DrivineDriftReportStore. The natural key carries `contextKey` rather + // than `contextId` because a Cypher MERGE can't key on a null, so a global report needs a + // non-null stand-in to be as idempotent as a scoped one. + UniquenessConstraintSpec( + label = "MetamodelDriftReport", + properties = listOf("schemaName", "versionHash", "capturedAt", "contextKey"), + ), + UniquenessConstraintSpec(label = "MetamodelDriftReportCounter", property = "schemaName"), + UniquenessConstraintSpec(label = "MetamodelDriftReport", properties = listOf("schemaName", "sequence")), + ) + + /** Every node label the metamodel stores write, for test cleanup and for drift exclusion. */ + val LABELS: List = listOf( + "MetamodelVersion", + "MetamodelSchemaCounter", + "MetamodelDriftReport", + "MetamodelDriftReportCounter", + ) +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt new file mode 100644 index 00000000..336e787c --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt @@ -0,0 +1,229 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import com.embabel.agent.core.ContextId +import com.embabel.dice.common.DiceMetadataKeys +import com.embabel.dice.metamodel.DeclaredSchema +import com.embabel.dice.metamodel.DeclaredSchemaSource +import com.embabel.dice.metamodel.DriftCheckRunner +import com.embabel.dice.metamodel.MetamodelVersion +import com.embabel.dice.metamodel.support.DefaultDriftCheckRunner +import com.embabel.dice.metamodel.support.MentionTypeDriftQuarantinePolicy +import com.embabel.dice.metamodel.support.StructuralMetamodelDiffer +import com.embabel.dice.proposition.EntityMention +import com.embabel.dice.proposition.MentionRole +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.proposition.PropositionStatus +import org.drivine.manager.PersistenceManager +import org.drivine.query.QuerySpecification +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource + +/** + * The whole drift check end to end, on real Drivine stores against a Neo4j testcontainer: + * `DefaultDriftCheckRunner` wired to [DrivineMetamodelVersionStore], [DrivineObservedSchemaSource], + * [DrivineDriftReportStore] and [DrivinePropositionRepository], with the real differ and the real + * quarantine policy. + * + * The unit tests in `dice-metamodel` already pin the runner's sequencing against fakes. What only a + * database can answer is whether the three persistent pieces line up: a report written by one store + * must name a hash the *other* store can resolve, and the proposition the policy flagged must come + * back out of the graph flagged. + */ +@SpringBootTest(classes = [TestApplication::class]) +class DrivineDriftCheckIntegrationTest { + + companion object { + @JvmStatic + @DynamicPropertySource + fun neo4jProperties(registry: DynamicPropertyRegistry) = Neo4jTestContainer.registerProperties(registry) + } + + @Autowired + private lateinit var versionStore: DrivineMetamodelVersionStore + + @Autowired + private lateinit var reportStore: DrivineDriftReportStore + + @Autowired + private lateinit var observedSchemaSource: DrivineObservedSchemaSource + + @Autowired + private lateinit var repository: DrivinePropositionRepository + + @Autowired + private lateinit var persistenceManager: PersistenceManager + + private val schemaName = "governed-schema" + private val contextId = ContextId("drift-ctx") + + /** Declares one entity type. Anything else the graph holds is drift. */ + private val declaredVersion = MetamodelVersion( + schemaName = schemaName, + entityTypeNames = listOf("Person"), + entityTypeLabels = mapOf("Person" to setOf("Person")), + entityTypeProperties = mapOf("Person" to emptySet()), + relationshipNames = emptyList(), + ) + + private val runner: DriftCheckRunner by lazy { + DefaultDriftCheckRunner( + declaredSchemaSource = DeclaredSchemaSource { + DeclaredSchema(version = declaredVersion, relationshipTypeNames = emptySet()) + }, + versionStore = versionStore, + observedSchemaSource = observedSchemaSource, + differ = StructuralMetamodelDiffer(), + driftReportStore = reportStore, + quarantinePolicy = MentionTypeDriftQuarantinePolicy(), + propositionStore = repository, + ) + } + + @AfterEach + fun cleanUp() { + repository.clearAll() + persistenceManager.execute(QuerySpecification.withStatement("MATCH (n) DETACH DELETE n")) + } + + @Test + fun `an undeclared mention type is reported, resolvable, and quarantined`() { + val stranded = repository.save( + Proposition( + contextId = contextId, + text = "The ghost haunts the manor", + mentions = listOf(EntityMention(span = "the ghost", type = "Ghost", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + + val result = runner.run(dryRun = false, contextId = contextId) + + // 1. The check saw the undeclared type and nothing else. + assertEquals(setOf("Ghost"), result.report.driftedEntityTypes) + assertEquals(1, result.quarantinedCount) + + // 2. The report is really in the graph, under the context it was scoped to. + val persisted = reportStore.driftReportsInContext(schemaName, contextId, limit = 10) + assertEquals(listOf(result.report), persisted) + assertTrue( + reportStore.globalDriftReports(schemaName, limit = 10).isEmpty(), + "a context-scoped check must not show up as a whole-graph one", + ) + + // 3. Its hash resolves through the *other* store -- the guarantee "stamp before you report" + // exists to buy, and the one that only breaks once both are real. + val resolved = versionStore.findVersion(schemaName, persisted.single().versionHash) + assertNotNull(resolved, "a persisted report named a version hash nothing recorded") + assertEquals(declaredVersion, resolved) + + // 4. The stranded proposition came back out of the graph flagged, with a reason a person + // can read. + val reloaded = repository.findById(stranded.id) + assertNotNull(reloaded) + assertEquals(PropositionStatus.STALE, reloaded!!.status) + val reason = reloaded.metadata[DiceMetadataKeys.QUARANTINE_REASON] as? String + assertNotNull(reason, "quarantine must say why; metadata was ${reloaded.metadata}") + assertTrue(reason!!.contains("Ghost"), "the reason must name the drifted type, but was: $reason") + } + + @Test + fun `a dry run records the same report and touches no proposition`() { + val untouched = repository.save( + Proposition( + contextId = contextId, + text = "The ghost is still here", + mentions = listOf(EntityMention(span = "the ghost", type = "Ghost", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + + val result = runner.run(dryRun = true, contextId = contextId) + + assertEquals(setOf("Ghost"), result.report.driftedEntityTypes) + assertEquals(0, result.quarantinedCount) + assertEquals(1, reportStore.driftReportsInContext(schemaName, contextId, limit = 10).size) + assertEquals(PropositionStatus.ACTIVE, repository.findById(untouched.id)!!.status) + } + + @Test + fun `a conforming context reports no drift, and the run is still on the record`() { + // A zero-drift check is a fact worth having, not a no-op -- "we looked and it was clean" is + // the answer an audit needs, and only a persisted report can give it. + repository.save( + Proposition( + contextId = contextId, + text = "Ada is a person", + mentions = listOf(EntityMention(span = "Ada", type = "Person", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + + val result = runner.run(dryRun = false, contextId = contextId) + + assertTrue(result.report.driftedEntityTypes.isEmpty(), "got ${result.report.driftedEntityTypes}") + assertEquals(0, result.quarantinedCount) + assertEquals(listOf(result.report), reportStore.driftReportsInContext(schemaName, contextId, limit = 10)) + } + + @Test + fun `two checks of one context accumulate as two reports, newest first`() { + repository.save( + Proposition( + contextId = contextId, + text = "The ghost haunts the manor", + mentions = listOf(EntityMention(span = "the ghost", type = "Ghost", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + + val first = runner.run(dryRun = true, contextId = contextId) + val second = runner.run(dryRun = true, contextId = contextId) + + val history = reportStore.driftReportsInContext(schemaName, contextId, limit = 10) + assertEquals(listOf(second.report, first.report), history, "a drift log accumulates; it is not a gauge") + } + + @Test + fun `another context's drift never reaches this one`() { + repository.save( + Proposition( + contextId = ContextId("elsewhere"), + text = "The poltergeist rattles the door", + mentions = listOf(EntityMention(span = "the poltergeist", type = "Poltergeist", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + + val result = runner.run(dryRun = false, contextId = contextId) + + assertTrue(result.report.driftedEntityTypes.isEmpty(), "got ${result.report.driftedEntityTypes}") + assertEquals(0, result.quarantinedCount) + assertEquals( + PropositionStatus.ACTIVE, + repository.findByContextId(ContextId("elsewhere")).single().status, + "a check scoped to one context must not be able to reach another's data", + ) + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt new file mode 100644 index 00000000..97a9d094 --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt @@ -0,0 +1,505 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import com.embabel.agent.core.ContextId +import com.embabel.dice.metamodel.DriftReport +import org.drivine.manager.PersistenceManager +import org.drivine.query.QuerySpecification +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import java.time.Instant + +/** + * [DrivineDriftReportStore] against a Neo4j testcontainer. Each test starts from an empty drift log + * via [cleanUp]. + * + * These are the same questions `DriftReportStoreTest` asks of the in-memory reference, answered in + * Cypher — plus the ones only a database can get wrong: the scope pushed into the query rather than + * applied to a page that has already been cut, a `since` bound that stays exact below the + * millisecond, and the sequence that keeps a limited page from moving between reads. + */ +@SpringBootTest(classes = [TestApplication::class]) +class DrivineDriftReportStoreIntegrationTest { + + companion object { + @JvmStatic + @DynamicPropertySource + fun neo4jProperties(registry: DynamicPropertyRegistry) = Neo4jTestContainer.registerProperties(registry) + } + + @Autowired + private lateinit var store: DrivineDriftReportStore + + @Autowired + private lateinit var persistenceManager: PersistenceManager + + private val schemaName = "drift-schema" + private val contextA = ContextId("ctx-a") + private val contextB = ContextId("ctx-b") + private val epoch: Instant = Instant.parse("2026-01-01T00:00:00Z") + + @AfterEach + fun cleanUp() { + listOf("MetamodelDriftReport", "MetamodelDriftReportCounter").forEach { label -> + persistenceManager.execute(QuerySpecification.withStatement("MATCH (n:$label) DETACH DELETE n")) + } + } + + /** A report captured [minute] minutes after the epoch, optionally scoped to a context. */ + private fun save( + minute: Long, + contextId: ContextId? = null, + schema: String = schemaName, + ): DriftReport { + val report = DriftReport( + schemaName = schema, + versionHash = "hash-$minute", + driftedEntityTypes = setOf("Ghost$minute"), + driftedRelationshipTypes = emptySet(), + capturedAt = epoch.plusSeconds(minute * 60), + contextId = contextId, + ) + store.saveDriftReport(report) + return report + } + + // ---- Round-trip ---- + + @Test + fun `a global report persists and reads back every field`() { + val report = DriftReport( + schemaName = schemaName, + versionHash = "abc123", + driftedEntityTypes = setOf("Ghost", "Phantom"), + driftedRelationshipTypes = setOf("HAUNTS"), + capturedAt = epoch.plusNanos(123_456_789), + ) + + store.saveDriftReport(report) + + assertEquals(listOf(report), store.driftReports(schemaName, limit = 10)) + assertEquals(listOf(report), store.globalDriftReports(schemaName, limit = 10)) + } + + @Test + fun `a context-scoped report keeps its context through the round-trip`() { + val report = DriftReport( + schemaName = schemaName, + versionHash = "abc123", + driftedEntityTypes = setOf("Ghost"), + driftedRelationshipTypes = emptySet(), + capturedAt = epoch, + contextId = contextA, + ) + + store.saveDriftReport(report) + + assertEquals(listOf(report), store.driftReportsInContext(schemaName, contextA, limit = 10)) + assertEquals(contextA, store.driftReports(schemaName, limit = 10).single().contextId) + } + + @Test + fun `type names carrying delimiter characters survive the round-trip`() { + // The sets are JSON, not a joined string, and these are the characters that would break a + // joined one. Type names come out of LLM extraction and do contain them. + val report = DriftReport( + schemaName = schemaName, + versionHash = "delimiters", + driftedEntityTypes = setOf("Type|WithPipe", "Type\tWithTab", "Type\nWithNewline", """Type"WithQuote"""), + driftedRelationshipTypes = setOf("""REL\WITH\BACKSLASH"""), + capturedAt = epoch, + ) + + store.saveDriftReport(report) + + assertEquals(report, store.driftReports(schemaName, limit = 10).single()) + } + + // ---- Ordering and bounding ---- + + @Test + fun `reads come back newest first by capture instant, not by write order`() { + // Written 1, 3, 2 -- so anything that ordered on the write sequence alone would hand back + // 2, 3, 1. The contract orders on when the graph was looked at. + save(1) + save(3) + save(2) + + assertEquals( + listOf("hash-3", "hash-2", "hash-1"), + store.driftReports(schemaName, limit = 10).map { it.versionHash }, + ) + } + + @Test + fun `a limit returns the newest page, not an arbitrary one`() { + (1L..5L).forEach { save(it) } + + val reports = store.driftReports(schemaName, limit = 2) + + assertEquals(listOf("hash-5", "hash-4"), reports.map { it.versionHash }) + } + + @Test + fun `since bounds the window from below, inclusively`() { + (1L..4L).forEach { save(it) } + + val reports = store.driftReports(schemaName, limit = 10, since = epoch.plusSeconds(2 * 60)) + + assertEquals(listOf("hash-4", "hash-3", "hash-2"), reports.map { it.versionHash }) + } + + @Test + fun `since stays exact below the millisecond`() { + // The reason the sort and the bound are stored as (epochSecond, nano) rather than as epoch + // milliseconds. These two are 500 microseconds apart, which truncates to the same + // millisecond: a millis-based bound would sweep the earlier one in and quietly widen the + // window the caller asked for. + val earlier = epoch.plusNanos(200_000) + val later = epoch.plusNanos(700_000) + listOf(earlier to "early", later to "late").forEach { (instant, hash) -> + store.saveDriftReport( + DriftReport(schemaName, hash, setOf("Ghost"), emptySet(), instant), + ) + } + + val fromLater = store.driftReports(schemaName, limit = 10, since = later) + + assertEquals(listOf("late"), fromLater.map { it.versionHash }) + assertEquals(2, store.driftReports(schemaName, limit = 10).size, "both were really stored") + assertEquals( + listOf("late", "early"), + store.driftReports(schemaName, limit = 10).map { it.versionHash }, + "and sub-millisecond ordering is right too", + ) + } + + @Test + fun `a non-positive limit is rejected by all three reads rather than quietly meaning everything`() { + save(1) + + listOf(0, -1).forEach { limit -> + assertThrows(IllegalArgumentException::class.java) { store.driftReports(schemaName, limit) } + assertThrows(IllegalArgumentException::class.java) { store.globalDriftReports(schemaName, limit) } + assertThrows(IllegalArgumentException::class.java) { + store.driftReportsInContext(schemaName, contextA, limit) + } + } + } + + @Test + fun `reports captured at the same instant hold a stable order across reads`() { + // A global sweep and a context sweep can share a capture instant. The instant alone then + // leaves their order to the database, and with a LIMIT on top the page boundary lands + // somewhere arbitrary -- the same read can return different rows each time. The per-schema + // sequence is what makes the order total, so a page is repeatable. + val instant = epoch.plusSeconds(600) + val first = DriftReport(schemaName, "tie-1", setOf("A"), emptySet(), instant, contextA) + val second = DriftReport(schemaName, "tie-2", setOf("B"), emptySet(), instant, contextB) + store.saveDriftReport(first) + store.saveDriftReport(second) + + val pages = (1..5).map { store.driftReports(schemaName, limit = 1).map { r -> r.versionHash } } + + assertEquals(List(5) { listOf("tie-2") }, pages, "a tie must not make the page wobble") + assertEquals(listOf("tie-2", "tie-1"), store.driftReports(schemaName, limit = 10).map { it.versionHash }) + } + + // ---- Scope ---- + + @Test + fun `each read sees only its own scope`() { + val global = save(1) + val inA = save(2, contextA) + val inB = save(3, contextB) + + assertEquals(listOf(inB, inA, global), store.driftReports(schemaName, limit = 10)) + assertEquals(listOf(global), store.globalDriftReports(schemaName, limit = 10)) + assertEquals(listOf(inA), store.driftReportsInContext(schemaName, contextA, limit = 10)) + assertEquals(listOf(inB), store.driftReportsInContext(schemaName, contextB, limit = 10)) + } + + @Test + fun `scoping happens before limiting, not after`() { + // The rule the contract exists to protect. A store that read a limited page and then + // filtered it would answer "no global drift" here: the newest three reports are all + // context-scoped, so the one global report never survives to the filter. + val global = save(1) + save(2, contextA) + save(3, contextA) + save(4, contextA) + + assertEquals(listOf(global), store.globalDriftReports(schemaName, limit = 3)) + } + + @Test + fun `a context read pushes its scope down too`() { + // The mirror image: the newest reports are global and another context's, so an in-memory + // filter over a page of 2 would report no drift in context A at all. + val inA = save(1, contextA) + save(2) + save(3, contextB) + + assertEquals(listOf(inA), store.driftReportsInContext(schemaName, contextA, limit = 2)) + } + + @Test + fun `reports for another schema are never returned`() { + save(1, schema = "other-schema") + + assertTrue(store.driftReports(schemaName, limit = 10).isEmpty()) + } + + // ---- Natural-key identity ---- + + @Test + fun `re-saving the same observation updates it in place and keeps its position`() { + val first = save(1) + save(2) + val corrected = DriftReport( + schemaName = first.schemaName, + versionHash = first.versionHash, + driftedEntityTypes = setOf("GhostA", "GhostB"), + driftedRelationshipTypes = setOf("HAUNTS"), + capturedAt = first.capturedAt, + contextId = first.contextId, + ) + + store.saveDriftReport(corrected) + + val reports = store.driftReports(schemaName, limit = 10) + assertEquals(2, reports.size, "same natural key means the same record, not a third one") + assertEquals(setOf("GhostA", "GhostB"), reports.last().driftedEntityTypes) + assertEquals(1L, storedSequence(first), "an idempotent re-save must not move it in the order") + assertEquals(2L, counterValue(schemaName), "nor consume a sequence number") + } + + @Test + fun `checks at different instants are separate records`() { + save(1) + save(2) + + assertEquals(2, store.driftReports(schemaName, limit = 10).size) + } + + @Test + fun `a global and a scoped check at the same instant are two records, not an overwrite`() { + // The reason the natural key carries a never-null contextKey. Keying on contextId itself + // would make the global report's key contain a null, which a Cypher MERGE can never match: + // it would take the CREATE branch every time and duplicate on retry. + val instant = epoch.plusSeconds(60) + val global = DriftReport(schemaName, "same-hash", setOf("Ghost"), emptySet(), instant) + val scoped = DriftReport(schemaName, "same-hash", setOf("Ghost"), emptySet(), instant, contextA) + + store.saveDriftReport(global) + store.saveDriftReport(scoped) + + assertEquals(2, store.driftReports(schemaName, limit = 10).size) + assertEquals(listOf(global), store.globalDriftReports(schemaName, limit = 10)) + assertEquals(listOf(scoped), store.driftReportsInContext(schemaName, contextA, limit = 10)) + } + + @Test + fun `a context named after the global marker cannot collide with a global report`() { + // ContextId accepts any non-blank string, so any encoding that stores a bare context id + // alongside a sentinel is one `ContextId(sentinel)` away from a collision: both reports + // would MERGE onto one node and each save would rewrite the other's scope, so a + // context-scoped finding would surface as whole-graph drift. Prefixing every real context + // makes that unrepresentable rather than unlikely -- including for the previous sentinel, + // which a caller may well still be using as a context id. + val instant = epoch.plusSeconds(60) + listOf( + ContextId(DriftReportRowMapper.GLOBAL_CONTEXT_KEY), + ContextId("\u0000__global-context__\u0000"), + ContextId("ctx:global"), + ).forEachIndexed { index, awkward -> + val capturedAt = instant.plusSeconds(index.toLong()) + val global = DriftReport(schemaName, "collide", setOf("G"), emptySet(), capturedAt) + val scoped = DriftReport(schemaName, "collide", setOf("S"), emptySet(), capturedAt, awkward) + + store.saveDriftReport(global) + store.saveDriftReport(scoped) + + assertEquals( + listOf(global), + store.globalDriftReports(schemaName, limit = 10, since = capturedAt), + "the global report kept its scope against context '${awkward.value}'", + ) + assertEquals( + listOf(scoped), + store.driftReportsInContext(schemaName, awkward, limit = 10, since = capturedAt), + "and the scoped report kept its own", + ) + } + } + + @Test + fun `saving one global report twice leaves one node, not two`() { + val report = DriftReport(schemaName, "idempotent", setOf("Ghost"), emptySet(), epoch) + + store.saveDriftReport(report) + store.saveDriftReport(report) + + assertEquals(listOf(report), store.globalDriftReports(schemaName, limit = 10)) + assertEquals(1L, counterValue(schemaName)) + } + + @Test + fun `two reports of one schema cannot be stored at the same position`() { + // The safety net under the sequence: whatever the counter does, the database will not hold + // two reports of one schema claiming one place in the tie-break order, so a lost counter + // update is a retryable failure rather than a silently wobbling page. + val first = save(1) + assertEquals(1L, storedSequence(first)) + + val collision = runCatching { + persistenceManager.execute( + QuerySpecification.withStatement( + """ + CREATE (n:MetamodelDriftReport { + schemaName: ${'$'}schemaName, versionHash: 'other', capturedAt: 'other', + contextKey: 'other', sequence: 1 + }) + """.trimIndent(), + ).bind(mapOf("schemaName" to schemaName)), + ) + } + + assertTrue(collision.isFailure, "the database must refuse a second report at position 1") + assertEquals(listOf(first), store.driftReports(schemaName, limit = 10), "and the log is untouched") + } + + // ---- Corrupt rows are skipped, not materialized ---- + + @Test + fun `a report node missing a required property is skipped and warned about by name`() { + val readable = save(1) + save(2) + persistenceManager.execute( + QuerySpecification.withStatement( + """ + MATCH (n:MetamodelDriftReport {schemaName: ${'$'}schemaName, versionHash: 'hash-2'}) + REMOVE n.driftedEntityTypes + """.trimIndent(), + ).bind(mapOf("schemaName" to schemaName)), + ) + + val (reports, logged) = capturingStoreWarnings { store.driftReports(schemaName, limit = 10) } + + assertEquals(listOf(readable), reports, "the readable report survives; the corrupt one is dropped") + assertTrue( + logged.any { + it.contains("Skipping unreadable MetamodelDriftReport row") && it.contains("driftedEntityTypes") + }, + "the skip must be warned about and name the missing property; warnings were: $logged", + ) + } + + @Test + fun `a report whose two halves of scope disagree is refused`() { + // contextKey is what a save MERGEs on; contextId is what a scoped read matches. A node + // where they disagree answers to one scope when read and a different one when re-saved. + save(1) + persistenceManager.execute( + QuerySpecification.withStatement( + """ + MATCH (n:MetamodelDriftReport {schemaName: ${'$'}schemaName}) + SET n.contextKey = 'not-the-global-sentinel' + """.trimIndent(), + ).bind(mapOf("schemaName" to schemaName)), + ) + + val (reports, logged) = capturingStoreWarnings { store.driftReports(schemaName, limit = 10) } + + assertTrue(reports.isEmpty()) + assertTrue(logged.any { it.contains("fails its scope check") }, "warnings were: $logged") + } + + @Test + fun `a node with no sort key is excluded in the database, not after the page is cut`() { + // Neo4j sorts null largest, so a node missing the sort key would sort to the front of a DESC + // order, spend one of the caller's limit slots, and then be dropped by the mapper -- hiding + // a perfectly good report behind a broken one. + val readable = save(1) + persistenceManager.execute( + QuerySpecification.withStatement( + """ + CREATE (n:MetamodelDriftReport { + schemaName: ${'$'}schemaName, versionHash: 'no-sort-key', capturedAt: 'never', + contextKey: ${'$'}globalKey, driftedEntityTypes: '[]', driftedRelationshipTypes: '[]' + }) + """.trimIndent(), + ).bind( + mapOf( + "schemaName" to schemaName, + "globalKey" to DriftReportRowMapper.GLOBAL_CONTEXT_KEY, + ), + ), + ) + + assertEquals(listOf(readable), store.driftReports(schemaName, limit = 1)) + } + + // ---- helpers ---- + + /** The `sequence` a report node actually holds — storage bookkeeping, so read straight out. */ + private fun storedSequence(report: DriftReport): Long? = persistenceManager.maybeGetOne( + QuerySpecification.withStatement( + """ + MATCH (n:MetamodelDriftReport {schemaName: ${'$'}schemaName, versionHash: ${'$'}versionHash}) + RETURN n.sequence AS sequence + """.trimIndent(), + ).bind(mapOf("schemaName" to report.schemaName, "versionHash" to report.versionHash)) + .transform(Long::class.java), + ) + + /** How far the schema's report counter has been advanced. */ + private fun counterValue(schemaName: String): Long? = persistenceManager.maybeGetOne( + QuerySpecification.withStatement( + "MATCH (c:MetamodelDriftReportCounter {schemaName: ${'$'}schemaName}) RETURN c.sequence AS sequence", + ).bind(mapOf("schemaName" to schemaName)).transform(Long::class.java), + ) + + /** + * Run [block] with a listener on the store's logger, and hand back both its result and every + * WARN it emitted. "Skips the row" and "skips the row *and says so*" are different behaviours, + * and only the second is any use to an operator. + */ + private fun capturingStoreWarnings(block: () -> T): Pair> { + val logger = LoggerFactory.getLogger(DrivineDriftReportStore::class.java) as Logger + val appender = ListAppender().apply { start() } + logger.addAppender(appender) + return try { + block() to appender.list.filter { it.level == Level.WARN }.map { it.formattedMessage } + } finally { + logger.detachAppender(appender) + appender.stop() + } + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt new file mode 100644 index 00000000..30d71c86 --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt @@ -0,0 +1,369 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import com.embabel.agent.core.ContextId +import com.embabel.dice.metamodel.DriftReport +import com.embabel.dice.metamodel.MetamodelVersion +import org.drivine.manager.PersistenceManager +import org.drivine.query.QuerySpecification +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import java.time.Instant + +/** + * [DrivineObservedSchemaSource] against a Neo4j testcontainer. + * + * The load-bearing test here is the last group: governance must never observe *itself*. Stamping a + * version and writing a drift report both add nodes to the very graph the next check looks at, so + * without the exclusion every run would report the previous run as drift and the noise would never + * settle. + */ +@SpringBootTest(classes = [TestApplication::class]) +class DrivineObservedSchemaSourceIntegrationTest { + + companion object { + @JvmStatic + @DynamicPropertySource + fun neo4jProperties(registry: DynamicPropertyRegistry) = Neo4jTestContainer.registerProperties(registry) + } + + @Autowired + private lateinit var source: DrivineObservedSchemaSource + + @Autowired + private lateinit var versionStore: DrivineMetamodelVersionStore + + @Autowired + private lateinit var reportStore: DrivineDriftReportStore + + @Autowired + private lateinit var persistenceManager: PersistenceManager + + private val tenantA = ContextId("tenant-a") + private val tenantB = ContextId("tenant-b") + + @AfterEach + fun cleanUp() { + persistenceManager.execute(QuerySpecification.withStatement("MATCH (n) DETACH DELETE n")) + } + + // ---- Context-scoped observation ---- + + @Test + fun `scoped entity types are the mention types of that context's propositions`() { + writeProposition("p-a", tenantA) + writeProposition("p-b", tenantB) + writeMention("p-a", "m-a", type = "Person") + writeMention("p-b", "m-b", type = "SecretType") + + assertEquals(setOf("Person"), source.observe(tenantA).entityTypeNames) + assertEquals(setOf("SecretType"), source.observe(tenantB).entityTypeNames) + } + + @Test + fun `a relationship sourced from one context is not seen from another`() { + writeProposition("p-a", tenantA) + writeProposition("p-b", tenantB) + writeRelationship(type = "ONLY_A_REL", sourcePropositionIds = listOf("p-a")) + + assertEquals(setOf("ONLY_A_REL"), source.observe(tenantA).relationshipTypeNames) + assertTrue(source.observe(tenantB).relationshipTypeNames.isEmpty()) + } + + @Test + fun `a relationship two contexts produced is drift in both of them`() { + // Deliberately does not collapse to one owner: an undeclared relationship type present in + // your context's data is drift in your context, whoever else also produced it. + writeProposition("p-a", tenantA) + writeProposition("p-b", tenantB) + writeRelationship(type = "SHARED_REL", sourcePropositionIds = listOf("p-a", "p-b")) + + assertTrue(source.observe(tenantA).relationshipTypeNames.contains("SHARED_REL")) + assertTrue(source.observe(tenantB).relationshipTypeNames.contains("SHARED_REL")) + } + + @Test + fun `relationship names come back exactly as stored, with no normalization`() { + writeProposition("p-a", tenantA) + writeRelationship(type = "Works_At_Company", sourcePropositionIds = listOf("p-a")) + + assertEquals(setOf("Works_At_Company"), source.observe(tenantA).relationshipTypeNames) + } + + @Test + fun `an empty context observes nothing rather than the whole graph`() { + writeProposition("p-a", tenantA) + writeMention("p-a", "m-a", type = "Person") + + val observed = source.observe(ContextId("nobody-here")) + + assertTrue(observed.entityTypeNames.isEmpty()) + assertTrue(observed.relationshipTypeNames.isEmpty()) + } + + // ---- Whole-graph observation ---- + + @Test + fun `whole-graph observation reports domain labels and relationship types`() { + writeProposition("p-a", tenantA) + writeRelationship(type = "GLOBAL_REL", sourcePropositionIds = listOf("p-a")) + + val observed = source.observe() + + assertTrue(observed.entityTypeNames.contains("Entity"), "got ${observed.entityTypeNames}") + assertTrue(observed.relationshipTypeNames.contains("GLOBAL_REL"), "got ${observed.relationshipTypeNames}") + } + + @Test + fun `an observation is stamped with the instant it was taken`() { + val before = Instant.now() + + val capturedAt = source.observe().capturedAt + + assertFalse(capturedAt.isBefore(before), "capturedAt $capturedAt predates the call at $before") + } + + // ---- Governance never observes its own bookkeeping ---- + + @Test + fun `dice's own storage labels are never reported as domain drift`() { + writeProposition("p-a", tenantA) + writeMention("p-a", "m-a", type = "Person") + persistenceManager.execute(QuerySpecification.withStatement("CREATE (:Source {key: 'src'})")) + persistenceManager.execute(QuerySpecification.withStatement("CREATE (:ProcessedChunk {id: 'chunk'})")) + // Without this the test could pass by observing nothing at all. The database really is + // reporting these labels; the exclusion is what keeps them out of the snapshot. + assertTrue( + rawLabels().containsAll(setOf("Proposition", "Mention", "Source", "ProcessedChunk")), + "precondition: the raw catalogue must hold the bookkeeping labels, but was ${rawLabels()}", + ) + + val observed = source.observe() + + assertTrue( + observed.entityTypeNames.none { it in DICE_BOOKKEEPING_LABELS }, + "bookkeeping leaked into the observation: ${observed.entityTypeNames intersect DICE_BOOKKEEPING_LABELS}", + ) + } + + @Test + fun `the metamodel's own nodes are never reported as domain drift`() { + // The self-reference that matters: a drift check stamps a version and writes a report, and + // both land in the graph the *next* check observes. Without the exclusion the second run + // reports MetamodelVersion and MetamodelDriftReport as undeclared entity types, and every + // run after that reports them again. + val version = MetamodelVersion("observed-schema", listOf("Person"), emptyMap(), emptyMap(), emptyList()) + versionStore.saveVersion(version) + reportStore.saveDriftReport( + DriftReport( + schemaName = "observed-schema", + versionHash = version.contentHash, + driftedEntityTypes = setOf("Ghost"), + driftedRelationshipTypes = emptySet(), + capturedAt = Instant.parse("2026-01-01T00:00:00Z"), + ), + ) + assertEquals(4, governanceNodeCount(), "precondition: all four governance node kinds exist") + assertTrue( + rawLabels().containsAll(MetamodelSchema.LABELS.toSet()), + "precondition: the raw catalogue must hold every governance label, but was ${rawLabels()}", + ) + + val observed = source.observe() + + MetamodelSchema.LABELS.forEach { label -> + assertFalse( + observed.entityTypeNames.contains(label), + "governance observed its own '$label' node as domain drift; got ${observed.entityTypeNames}", + ) + } + } + + @Test + fun `dice's own relationship types are never reported as domain drift`() { + // HAS_MENTION and DERIVED_FROM sit on every proposition ever stored, so this is not an edge + // case: without the exclusion, the first whole-graph check against a populated graph + // reports them as relationship drift, and so does every check after it. + writeProposition("p-a", tenantA) + writeMention("p-a", "m-a", type = "Person") + persistenceManager.execute( + QuerySpecification.withStatement( + "MATCH (p:Proposition {id: 'p-a'}) CREATE (p)-[:DERIVED_FROM]->(:Source {key: 'src'})", + ), + ) + assertTrue( + rawRelationshipTypes().containsAll(setOf("HAS_MENTION", "DERIVED_FROM")), + "precondition: the raw catalogue must hold the bookkeeping edges, but was ${rawRelationshipTypes()}", + ) + + val observed = source.observe() + + assertTrue( + observed.relationshipTypeNames.none { it in DICE_BOOKKEEPING_RELATIONSHIP_TYPES }, + "bookkeeping edges leaked: " + + "${observed.relationshipTypeNames intersect DICE_BOOKKEEPING_RELATIONSHIP_TYPES}", + ) + } + + // ---- a name is not a reservation ---- + + @Test + fun `a domain node that only shares a bookkeeping label's name is still observed`() { + // The failure this guards against: reserving the *name* `Source` would mean an app that + // genuinely governs a type called Source could never see it reported as undeclared. A drift + // check that structurally cannot report a type is worse than one that reports too much. + persistenceManager.execute( + QuerySpecification.withStatement("CREATE (:Source {companyName: 'Acme', founded: 1999})"), + ) + assertTrue(rawLabels().contains("Source")) + + val observed = source.observe() + + assertTrue( + observed.entityTypeNames.contains("Source"), + "a Source node with none of dice's shape is domain data; got ${observed.entityTypeNames}", + ) + } + + @Test + fun `dice's own nodes are still excluded when nothing else claims their label`() { + // The other half: the shape test must not have simply stopped excluding anything. + writeProposition("p-a", tenantA) + writeMention("p-a", "m-a", type = "Person") + persistenceManager.execute(QuerySpecification.withStatement("CREATE (:Source {key: 'src-1'})")) + persistenceManager.execute(QuerySpecification.withStatement("CREATE (:ProcessedChunk {id: 'chunk-1'})")) + + val observed = source.observe() + + listOf("Proposition", "Mention", "Source", "ProcessedChunk").forEach { label -> + assertFalse( + observed.entityTypeNames.contains(label), + "dice's own '$label' nodes match its shape and must stay excluded; got ${observed.entityTypeNames}", + ) + } + } + + @Test + fun `a domain relationship sharing a bookkeeping type's name is still observed`() { + // Same rule on the relationship side, decided by the same marker the scoped path uses: + // an edge carrying sourcePropositions was projected from domain data, whatever it is called. + writeProposition("p-a", tenantA) + writeRelationship(type = "DERIVED_FROM", sourcePropositionIds = listOf("p-a")) + + assertTrue( + source.observe().relationshipTypeNames.contains("DERIVED_FROM"), + "got ${source.observe().relationshipTypeNames}", + ) + assertTrue( + source.observe(tenantA).relationshipTypeNames.contains("DERIVED_FROM"), + "the scoped path must agree; got ${source.observe(tenantA).relationshipTypeNames}", + ) + } + + @Test + fun `every label the trace and metamodel stores write has a shape entry`() { + // Keeps the shape map one edit away from the schema objects: a new node label added to + // either store without a shape here would silently stop being excluded. + (CollectorTraceSchema.LABELS + MetamodelSchema.LABELS).forEach { label -> + assertTrue( + DICE_BOOKKEEPING_LABEL_SHAPES.containsKey(label), + "'$label' is written by a dice store but has no bookkeeping shape", + ) + } + } + + // ---- helpers ---- + + /** + * A proposition node carrying dice's full shape, because the shape is what tells the observer + * this is dice's node and not a domain one that happens to share the label. + */ + private fun writeProposition(id: String, contextId: ContextId) { + persistenceManager.execute( + QuerySpecification.withStatement( + "CREATE (:Proposition {id: \$id, contextId: \$contextId, text: \$text})", + ).bind(mapOf("id" to id, "contextId" to contextId.value, "text" to "a fact about $id")), + ) + } + + /** Likewise a mention: `id`, `span`, `type` and `role`, as `PropositionGraphMapper` writes it. */ + private fun writeMention(propositionId: String, mentionId: String, type: String) { + persistenceManager.execute( + QuerySpecification.withStatement( + """ + MATCH (p:Proposition {id: ${'$'}propositionId}) + CREATE (p)-[:HAS_MENTION]->( + :Mention {id: ${'$'}mentionId, span: ${'$'}span, type: ${'$'}type, role: 'SUBJECT'} + ) + """.trimIndent(), + ).bind( + mapOf( + "propositionId" to propositionId, + "mentionId" to mentionId, + "span" to type.lowercase(), + "type" to type, + ), + ), + ) + } + + /** + * A projected domain edge, carrying the `sourcePropositions` property the graph writer stamps + * on everything it persists. APOC, because the type is a parameter and Cypher can't take one + * literally. + */ + private fun writeRelationship(type: String, sourcePropositionIds: List) { + persistenceManager.execute( + QuerySpecification.withStatement( + """ + MERGE (a:Entity {id: 'e1'}) + MERGE (b:Entity {id: 'e2'}) + WITH a, b + CALL apoc.create.relationship(a, ${'$'}relType, {sourcePropositions: ${'$'}sources}, b) YIELD rel + RETURN rel + """.trimIndent(), + ).bind(mapOf("relType" to type, "sources" to sourcePropositionIds)), + ) + } + + /** What the database's own catalogue says, before the source subtracts anything from it. */ + private fun rawLabels(): Set = queryStrings("CALL db.labels() YIELD label RETURN label") + + private fun rawRelationshipTypes(): Set = + queryStrings("CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType") + + private fun queryStrings(statement: String): Set = persistenceManager + .query(QuerySpecification.withStatement(statement).transform(String::class.java)) + .filterNotNull() + .toSet() + + private fun governanceNodeCount(): Int = MetamodelSchema.LABELS.count { label -> + ( + persistenceManager.maybeGetOne( + QuerySpecification.withStatement("MATCH (n:$label) RETURN count(n) AS c") + .transform(Long::class.java), + ) ?: 0L + ) > 0L + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt index 1abdcec8..56641468 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt @@ -159,21 +159,20 @@ open class TestApplication { ): GraphDecayManager = GraphDecayManager(repository, persistenceManager) /** - * Both MERGEs the version store performs need their key to be unique, because a MERGE is - * race-free only then. Without the first, concurrent saves of one version all miss the match, - * all create, and the history fills with duplicates. Without the second, a schema can end up - * with two counter nodes handing out the same sequence numbers. + * Every MERGE the governance stores perform needs its key to be unique, because a MERGE is + * race-free only then. Without that, concurrent saves of one record all miss the match, all + * create, and the history fills with duplicates. That covers the version node, the drift-report + * node, and both counter nodes; a schema with two counter nodes hands out the same sequence + * numbers twice. * - * The third backs the sequence itself: it makes two versions of one schema sharing a position - * impossible to store, so a lost counter update fails with a constraint violation the caller - * can retry. `DrivineMetamodelVersionStoreIntegrationTest` pins all three. + * The two `sequence` constraints back the ordering. They make two records of one schema sharing + * a position impossible to store, so a lost counter update fails with a constraint violation the + * caller can retry. `DrivineMetamodelVersionStoreIntegrationTest` and + * `DrivineDriftReportStoreIntegrationTest` pin them. The list lives in [MetamodelSchema], which + * keeps the constraints and the label list the observed-schema source excludes in one place. */ @Bean - open fun metamodelSchema(): SchemaCatalog = SchemaCatalog.of( - UniquenessConstraintSpec(label = "MetamodelVersion", properties = listOf("schemaName", "contentHash")), - UniquenessConstraintSpec(label = "MetamodelSchemaCounter", property = "schemaName"), - UniquenessConstraintSpec(label = "MetamodelVersion", properties = listOf("schemaName", "sequence")), - ) + open fun metamodelSchema(): SchemaCatalog = SchemaCatalog.of(MetamodelSchema.specs()) @Bean open fun metamodelClock(): PinnableClock = PinnableClock() @@ -183,4 +182,19 @@ open class TestApplication { persistenceManager: PersistenceManager, clock: PinnableClock, ): DrivineMetamodelVersionStore = DrivineMetamodelVersionStore(persistenceManager, clock) + + @Bean + open fun driftReportStore( + persistenceManager: PersistenceManager, + ): DrivineDriftReportStore = DrivineDriftReportStore(persistenceManager) + + /** + * On the system clock, not [PinnableClock]. An observation's capture instant is half of a drift + * report's identity, so two checks sharing one really are the same record — pinning here by + * default would silently collapse consecutive checks into a single report. + */ + @Bean + open fun observedSchemaSource( + persistenceManager: PersistenceManager, + ): DrivineObservedSchemaSource = DrivineObservedSchemaSource(persistenceManager) } diff --git a/docs/design/architecture.md b/docs/design/architecture.md index fc362ee0..bd7e3a37 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -12,7 +12,7 @@ DICE is a multi-module Maven build. Each module's intent, and what it's allowed | Module | Intent | |---|---| | `dice` | The core: proposition model, pipeline, gates, projection interfaces, query facades, agent tools, REST controllers. In-memory implementations only — no database driver. | -| `dice-storage` | The durable Neo4j backend: `Drivine`-based repository, graph/Prolog/lineage projectors, schema and index bootstrap, `MetamodelVersionStore` persistence. Depends on `dice` and `dice-metamodel`. | +| `dice-storage` | The durable Neo4j backend: `Drivine`-based repository, graph/Prolog/lineage projectors, schema and index bootstrap, and the whole governance persistence side — `MetamodelVersionStore`, the `DriftReportStore` drift log, and the `ObservedSchemaSource` that asks the live graph what it actually holds (excluding dice's own bookkeeping labels and edges, so governance never observes itself). Depends on `dice` and `dice-metamodel`. | | `dice-storage-autoconfigure` | Spring Boot autoconfiguration that wires `dice-storage`'s beans (repository, projectors, trust scorer) into a host application. Depends on `dice-storage`. | | `dice-ingestion` | Content-hash dedup ledger and source adapters that sit in front of `PropositionPipeline`, so the same artifact is never extracted twice concurrently. Depends on `dice`. | | `dice-report` | Rationale and structured report generation over propositions and their lineage. Depends on `dice`. | diff --git a/docs/design/metamodel-drift.md b/docs/design/metamodel-drift.md index ace62331..3cd47f96 100644 --- a/docs/design/metamodel-drift.md +++ b/docs/design/metamodel-drift.md @@ -614,20 +614,61 @@ once, is fine. Two concurrent checks of the same schema don't corrupt anything, its own complete snapshot, but they duplicate work; serialize at the scheduling layer if that matters. +## Persistence + +Both storage-side contracts are implemented in `dice-storage`, against Neo4j via Drivine. + +`DrivineDriftReportStore` keeps each check as a `(:MetamodelDriftReport)` node, MERGEd on the +natural key `(schemaName, versionHash, capturedAt, contextKey)` — where `contextKey` is `global` for +an unscoped check and `ctx:` for a scoped one, because a Cypher MERGE cannot key on a null. The +prefix is what makes the encoding injective: `ContextId` accepts any non-blank string, so an +unprefixed id plus a bare sentinel would let a context named after the sentinel share a key with the +global bucket and silently rewrite its scope. Each of the three bounded reads is +its own statement with its scope in the `WHERE` clause and the `LIMIT` after it, which is the whole +point: filtering a page that has already been cut applies the limit before the scope and can report +zero global drift while plenty sits in the store. Reports come back newest first by capture instant, +compared to the nanosecond so a `since` window stays exact, with a per-schema counter breaking exact +ties so a limited page doesn't move between reads. + +`DrivineObservedSchemaSource` takes the snapshot. Unscoped, it reads the database's own catalogue +(`db.labels()`, `db.relationshipTypes()`); scoped to a context, it derives entity types from that +context's mentions and relationship types from the `sourcePropositions` each projected edge carries. +Either way it subtracts dice's own bookkeeping — the proposition, mention, provenance, lineage, +collector-trace and metamodel node labels, and the `HAS_MENTION`/`DERIVED_FROM`-style edges. That +subtraction is load-bearing rather than tidy: stamping a version and writing a report both add nodes +to the very graph the next check looks at, so without it every run would report the previous run as +drift and the noise would never settle. + +**The subtraction is by shape, not by name.** Reserving the name `Source` would mean an app that +genuinely governs a type called `Source` could never see it reported as undeclared — a drift check +that structurally cannot report a type is worse than one that reports too much. So a bookkeeping +label is excluded only while every node carrying it matches dice's shape for it (dice's `Source` +nodes carry `key`, its governance nodes carry `schemaName`, and so on), and a bookkeeping +relationship type only while no edge of that type carries `sourcePropositions` — the marker the +graph writer stamps on every edge it projects from domain data, and the same one the context-scoped +query already selects on. + +Two limits are worth stating rather than papering over. Exclusion is decided per label, not per +node, so a graph mixing a domain `Source` with dice's own reports `Source` every run until the type +is declared — deliberately the visible direction to fail in. And deciding it costs a scan of dice's +own labels on each unscoped observation; context-scoped checks never pay it. + +Hosts declare the constraints these stores need (see `MetamodelSchema`); a MERGE is only race-free +under a uniqueness constraint on the key it merges on. + ## What comes next -`DriftReportStore` and `ObservedSchemaSource` are contracts here with no implementation yet. They -need a Drivine-backed report store — one that persists a report's `declaredDiff` alongside its drifted -type sets — and an observer that asks Neo4j for its distinct labels and relationship types. +`DriftReportStore` and `ObservedSchemaSource` now have Drivine implementations in `dice-storage`. -`DriftSweepCapable` and `SweptBaselineStore` are the same: contracts with an in-memory reference +`DriftSweepCapable` and `SweptBaselineStore` are contracts with an in-memory reference implementation and no durable one. Until the graph-backed store implements `SweptBaselineStore`, a Drivine-backed host gets the graph-truth half of a report and a `null` declared comparison. Until it implements `DriftSweepCapable`, a host sweeps through `PropositionStoreDriftSweep`, which is correct and does its filtering in the JVM. -There is no Spring configuration in `dice-metamodel` either, so a runner is an ordinary constructor -call, and nothing sweeps unless a host calls it. +There is no Spring configuration in `dice-metamodel` or `dice-storage`, so a runner is an ordinary +constructor call until the autoconfigure slice assembles one, and nothing sweeps unless a host +calls it. **Registration-time compatibility evaluation** is deferred design, tracked under the metamodel epic (`embabel/dice#45`) until it gets its own issue. A registry-style compatibility check would grade a From c4c46a9ceba5773664d9b707047fb853beaee4f8 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:16:21 -0400 Subject: [PATCH 02/10] docs(dice-storage): voice pass on drift-store docs and KDoc Comment and doc text only; no code change. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- CHANGELOG.md | 41 +++--- .../dice/storage/DrivineDriftReportStore.kt | 103 +++++++------- .../storage/DrivineObservedSchemaSource.kt | 129 ++++++++---------- .../dice/storage/MetamodelRowMappers.kt | 8 +- .../embabel/dice/storage/MetamodelSchema.kt | 27 ++-- .../DrivineDriftCheckIntegrationTest.kt | 20 +-- .../DrivineDriftReportStoreIntegrationTest.kt | 74 +++++----- ...vineObservedSchemaSourceIntegrationTest.kt | 49 ++++--- .../embabel/dice/storage/TestApplication.kt | 6 +- docs/design/architecture.md | 2 +- docs/design/metamodel-drift.md | 53 ++++--- 11 files changed, 242 insertions(+), 270 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc475d5d..5804d8c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -347,29 +347,28 @@ and the consumer PRs that deliver it). and no existing signature changed. - Drivine/Neo4j-backed drift persistence in `dice-storage`. `DrivineDriftReportStore` keeps each check as a `(:MetamodelDriftReport)` node, MERGEd on the natural key - `(schemaName, versionHash, capturedAt, contextKey)` — `contextKey` being `global` or `ctx:`, - since a Cypher MERGE cannot key on a null; prefixing every real context keeps the encoding - injective, so no `ContextId` value can share a key with the global bucket and silently rewrite its - scope. All three bounded reads are separate statements that push their scope - into the query *before* the `LIMIT`, which is what stops a schema whose recent history is mostly - context-scoped from reporting zero global drift while plenty sits in the store. Ordering is newest - first by capture instant, stored as `(epochSecond, nano)` so both the sort and an inclusive `since` - window stay exact below the millisecond, with a per-schema `(:MetamodelDriftReportCounter)` - sequence breaking exact ties so a limited page is repeatable. `DrivineObservedSchemaSource` takes - the snapshot: `db.labels()`/`db.relationshipTypes()` unscoped, and per context the distinct mention - types plus the edges whose `sourcePropositions` name that context's propositions. The unscoped - path subtracts dice's own bookkeeping — every proposition, provenance, lineage, collector-trace - and metamodel node label, and the `HAS_MENTION`/`DERIVED_FROM`/`SCORED`/`RETIRED_IN` edges — so - governance never observes the nodes its own last run wrote as domain drift. That subtraction is by - *shape*, not by name: a label is hidden only while every node carrying it matches dice's shape for - it, and an edge type only while none of its edges carries `sourcePropositions`, so an app - governing a type genuinely called `Source` still sees it reported. `MetamodelSchema` - collects the uniqueness constraints these stores need alongside the label list the observer - excludes, so the two cannot drift apart. Still no Spring wiring; that arrives in the autoconfigure - slice. + `(schemaName, versionHash, capturedAt, contextKey)`, with `contextKey` either `global` or + `ctx:`, since a Cypher MERGE cannot key on a null. Prefixing every real context keeps the + encoding injective, so no `ContextId` value can share a key with the global bucket and rewrite its + scope. All three bounded reads are separate statements that push their scope into the query ahead + of the `LIMIT`, which stops a schema whose recent history is mostly context-scoped from reporting + zero global drift while plenty sits in the store. Ordering is newest first by capture instant, + stored as `(epochSecond, nano)` so both the sort and an inclusive `since` window stay exact below + the millisecond, with a per-schema `(:MetamodelDriftReportCounter)` sequence breaking exact ties so + a limited page is repeatable. `DrivineObservedSchemaSource` takes the snapshot: + `db.labels()`/`db.relationshipTypes()` unscoped, and per context the distinct mention types plus + the edges whose `sourcePropositions` name that context's propositions. The unscoped path subtracts + dice's own bookkeeping — every proposition, provenance, lineage, collector-trace and metamodel node + label, and the `HAS_MENTION`/`DERIVED_FROM`/`SCORED`/`RETIRED_IN` edges — which keeps governance + from observing the nodes its own last run wrote as domain drift. That subtraction goes by node + shape: a label is hidden only while every node carrying it matches dice's shape for it, and an edge + type only while none of its edges carries `sourcePropositions`, so an app governing a type called + `Source` still sees it reported. `MetamodelSchema` collects the uniqueness constraints these stores + need alongside the label list the observer excludes, keeping the two in one place. Still no Spring + wiring; that arrives in the autoconfigure slice. **Compatibility: additive.** New classes in an existing module; no existing API touched. Hosts that already declared the three `MetamodelVersion`/`MetamodelSchemaCounter` constraints by hand can - swap in `MetamodelSchema.specs()`, which is a superset — the drift-report store needs three more: + swap in `MetamodelSchema.specs()`, a superset. The drift-report store needs three more: `MetamodelDriftReport(schemaName, versionHash, capturedAt, contextKey)`, `MetamodelDriftReportCounter(schemaName)`, and `MetamodelDriftReport(schemaName, sequence)`. - Optional source revisions in the `dice` core provenance model, the first slice of DICE #64. diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt index 87ed04ea..7d5ff385 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt @@ -29,42 +29,39 @@ import java.time.Instant * `(:MetamodelDriftReport)` node. * * The write MERGEs on the natural key `(schemaName, versionHash, capturedAt, contextKey)`, so a - * retry writes the same node again rather than a second copy of one observation. That is only - * race-free under a uniqueness constraint on the same four properties — see [MetamodelSchema], and - * see [DriftReportRowMapper.GLOBAL_CONTEXT_KEY] for why the fourth is `contextKey` and not - * `contextId`. + * retry updates the same node in place. That is race-free only under a uniqueness constraint on the + * same four properties; see [MetamodelSchema], and see [DriftReportRowMapper.GLOBAL_CONTEXT_KEY] for + * why the fourth property is `contextKey`. * - * Every statement is parameterized; nothing caller-derived is ever interpolated into Cypher. + * Every statement is parameterized; nothing caller-derived is interpolated into Cypher. * - * ## Scope goes into the query, never into a filter afterwards + * ## Scope is applied in the query, ahead of the limit * - * The contract's hardest rule, and the one its test pins: each of the three reads has its own - * statement, with its scope in the `WHERE` clause and the `LIMIT` applied after it. The tempting - * shortcut — read one limited page and filter it down in Kotlin — applies the limit *before* the - * scope, so a schema whose recent history happens to be mostly context-scoped would report zero - * global drift while plenty sat in the store. That is a wrong answer that looks exactly like a - * right one, which is why there are three statements here rather than one and a `filter`. + * Each of the three reads has its own statement, with its scope in the `WHERE` clause and the + * `LIMIT` applied after it. Reading one limited page and filtering it in Kotlin would apply the + * limit before the scope, so a schema whose recent history is mostly context-scoped would report + * zero global drift while the store held plenty. Hence three statements, and no `filter`. The + * contract's test pins this. * - * ## Newest first means the capture instant, broken by a counter + * ## Ordering: capture instant, with a per-schema counter breaking ties * - * The order is [DriftReport.capturedAt] descending — the instant the graph was *looked at*, which - * is what the contract promises and what a `since` window bounds. Write order can't stand in for - * it: a check of last week's snapshot saved today is still last week's observation. + * The order is [DriftReport.capturedAt] descending, the instant the graph was looked at, which is + * what the contract promises and what a `since` window bounds. Write order can't stand in for it: a + * check of last week's snapshot saved today is still last week's observation. * - * The instant alone is not a total order, though. Two reports of one schema — a global sweep and a - * per-context one, say — can share a capture instant, and then a plain `ORDER BY` leaves their - * relative order to the database. With a `LIMIT` on top that is not merely untidy: the page - * boundary lands somewhere arbitrary, so the same read can return different rows each time and a - * caller walking the history can miss one entirely. So each report also takes the next value off a - * per-schema `(:MetamodelDriftReportCounter)` node when — and only when — its node is first - * created, and that sequence breaks ties. Same mechanism as - * [DrivineMetamodelVersionStore]'s, deliberately on its own counter node: stamps and reports have - * very different volumes (a stamp per schema change, a report per scheduled check), a shared - * counter would put every drift check in the same run into a write conflict with the version stamp - * that precedes it, and the version store's own sequence would grow gaps that mean nothing. + * The instant alone is not a total order. Two reports of one schema, say a global sweep and a + * per-context one, can share a capture instant, and a plain `ORDER BY` then leaves their relative + * order to the database. Under a `LIMIT` the page boundary lands arbitrarily, so the same read can + * return different rows each time and a caller walking the history can miss one. So each report also + * takes the next value off a per-schema `(:MetamodelDriftReportCounter)` node, only when its node is + * first created, and that sequence breaks ties. [DrivineMetamodelVersionStore] uses the same + * mechanism on its own counter node. Stamps and reports have very different volumes — a stamp per + * schema change, a report per scheduled check — so a shared counter would put every drift check in a + * run into a write conflict with the version stamp preceding it, and would leave gaps in the version + * store's sequence. * - * A re-save of a report that already exists neither bumps the counter nor reassigns the sequence, - * so an idempotent write stays idempotent and a corrected observation keeps its original place. + * A re-save of an existing report leaves the counter and the sequence alone, so the write stays + * idempotent and a corrected observation keeps its original place. * * @param persistenceManager Drivine's handle on the `neo` datasource. */ @@ -82,17 +79,16 @@ open class DrivineDriftReportStore( * exists without its place in the tie-break order. * * `WITH n WHERE n.sequence IS NULL` separates the two halves. On a re-save that filters the - * row away, so the counter is never bumped and the existing sequence is never reassigned — - * the drifted type sets are refreshed and the report keeps the position it has always had. + * row away, leaving the counter and the existing sequence untouched; the drifted type sets + * are refreshed and the report keeps its original position. * * `SET c.lockedBy = ...` writes a property nobody reads, to take the exclusive lock on the - * counter before the increment below reads it; on its own the increment is a - * read-modify-write and two concurrent savers could both read 5 and both write 6. It is - * cheap insurance rather than a proven necessity — the same measurement in - * [DrivineMetamodelVersionStore] could not tell the locked and unlocked versions apart. - * What is load-bearing is the uniqueness constraint on `(schemaName, sequence)`: a lost - * update becomes a loud, retryable constraint violation instead of a silently arbitrary - * order. + * counter before the increment below reads it. On its own the increment is a + * read-modify-write, and two concurrent savers could both read 5 and both write 6. The + * benefit is unproven: the same measurement in [DrivineMetamodelVersionStore] could not tell + * the locked and unlocked versions apart. The guarantee comes from the uniqueness constraint + * on `(schemaName, sequence)`, under which a lost update fails with a constraint violation + * the caller can retry. */ private val SAVE_REPORT = """ MERGE (n:MetamodelDriftReport { @@ -119,11 +115,10 @@ open class DrivineDriftReportStore( /** * Every read starts here. * - * `capturedAtEpochSecond IS NOT NULL` is not defensive noise. Neo4j sorts null as the - * *largest* value, so a node missing the sort key would sort to the front of a DESC order, - * consume a slot of the caller's `limit`, and then be dropped by the mapper — hiding a - * perfectly good report behind a broken one. A node with no sort key never took a place in - * the order at all, so it is excluded in the database instead. + * `capturedAtEpochSecond IS NOT NULL` is load-bearing. Neo4j sorts null as the largest + * value, so a node missing the sort key would sort to the front of a DESC order, consume a + * slot of the caller's `limit`, and then be dropped by the mapper, hiding a good report + * behind a broken one. Excluding it in the database keeps it out of the order entirely. */ private val MATCH_SCHEMA = """ MATCH (n:MetamodelDriftReport {schemaName: ${'$'}schemaName}) @@ -133,14 +128,14 @@ open class DrivineDriftReportStore( /** Only unscoped, whole-graph checks: a global report has no `contextId` property at all. */ private const val ONLY_GLOBAL = "AND n.contextId IS NULL" - /** Only one context's checks. Global reports and other contexts' are both excluded. */ + /** Only one context's checks. Global reports and other contexts' are excluded. */ private const val ONLY_CONTEXT = "AND n.contextId = \$contextId" /** * The `since` bound, inclusive, compared second-then-nanosecond so it is exact. * - * Comparing a single truncated millisecond value would be simpler and subtly wrong: a bound - * falling part-way through a millisecond would sweep in reports captured just before it. + * Comparing a single truncated millisecond value would sweep in reports captured just before + * a bound that falls part-way through a millisecond. */ private val SINCE_BOUND = """ AND (n.capturedAtEpochSecond > ${'$'}sinceEpochSecond @@ -194,15 +189,15 @@ open class DrivineDriftReportStore( /** * Assemble and run one of the three scoped reads. * - * The statement is built from the constants above and nothing else — `scope` is one of this - * class's own literals, never anything a caller supplied — so this stays string *assembly*, not - * string interpolation of user data. Every value still travels as a bound parameter. + * The statement is assembled from the constants above and nothing else. `scope` is one of this + * class's own literals, so no caller-supplied text reaches the statement, and every value + * travels as a bound parameter. * - * A corrupt row that survives the query is warned about and skipped rather than taking down the - * whole governance read; see [DriftReportRowMapper], which throws instead of inventing defaults - * precisely so this can happen. Note the honest consequence of a bounded read: a skipped row has - * already spent one of the caller's `limit` slots, so a page can come back shorter than asked - * for. Silently reading further to backfill would break the bound the contract exists to keep. + * A corrupt row that survives the query is logged and skipped, which keeps one bad node from + * failing the whole governance read; [DriftReportRowMapper] throws rather than inventing + * defaults so that this can happen. A skipped row has already spent one of the caller's `limit` + * slots, so a page can come back shorter than asked for. Reading further to backfill would break + * the bound the contract keeps. */ private fun readPage( scope: String?, diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt index bfb89ef7..f41dd9fe 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt @@ -23,27 +23,23 @@ import org.drivine.query.QuerySpecification import java.time.Clock /** - * Every node label dice writes for its own bookkeeping, and the properties that identify a node as - * really being dice's rather than merely wearing the same label. + * Every node label dice writes for its own bookkeeping, and the properties that identify a node + * carrying that label as dice's own. * - * A drift check compares the *domain* schema an app declared against what a live graph actually - * holds. None of these labels was ever part of anybody's declared domain schema, so counting them - * would flag dice's own storage machinery as drift on every run — and, worse, would make governance - * report *itself*, since stamping a version and writing a report both add labels to the very graph - * the next check observes. A check that fires because the last check ran is noise that never - * settles. + * A drift check compares the domain schema an app declared against what a live graph holds. None of + * these labels belongs to a declared domain schema, so counting them would flag dice's own storage + * as drift on every run, and would make governance report itself: stamping a version and writing a + * report both add labels to the graph the next check observes. * - * **The shape is here because a name is not a reservation.** Excluding the *name* `Source` would - * hide a domain type genuinely called `Source` — an undeclared one could then never appear in an - * unscoped report, which is the failure a drift check exists to prevent, made permanent and silent. - * So a label is only excluded when the nodes carrying it actually look like dice's: dice's `Source` - * nodes carry `key`, its governance nodes carry `schemaName`, and so on. A same-named node that - * doesn't match keeps the label in the observation. + * Exclusion is by shape, so that a domain type called `Source` stays visible. A label is excluded + * only when the nodes carrying it match dice's shape for it — dice's `Source` nodes carry `key`, its + * governance nodes carry `schemaName`, and so on. A same-named node that doesn't match keeps the + * label in the observation, where an undeclared type can still be reported. * - * Each shape is the label's declared uniqueness key, extended with properties the node fragment - * writes unconditionally where those add discrimination. Deliberately kept *minimal*: a shape that - * demanded a property dice doesn't always write would make dice's own nodes look foreign and - * reintroduce the self-reporting bug, which is the worse direction to be wrong in. + * Each shape is the label's declared uniqueness key, plus properties the node fragment writes + * unconditionally where those add discrimination. Shapes are kept minimal: one demanding a property + * dice doesn't always write would make dice's own nodes look foreign and bring back the + * self-reporting case above. * * `DiceBookkeepingShapeTest` pins that every label in [CollectorTraceSchema.LABELS] and * [MetamodelSchema.LABELS] has an entry here, so adding a node label to either store can't quietly @@ -82,17 +78,16 @@ val DICE_BOOKKEEPING_LABELS: Set = DICE_BOOKKEEPING_LABEL_SHAPES.keys /** * Relationship types dice writes for its own bookkeeping, on the same grounds as - * [DICE_BOOKKEEPING_LABEL_SHAPES] and subject to the same name-is-not-a-reservation rule. + * [DICE_BOOKKEEPING_LABEL_SHAPES] and under the same shape rule. * - * Worth stating plainly, because the obvious first version of this class had no such set: dice's - * bookkeeping is *not* all node labels. `HAS_MENTION` and `DERIVED_FROM` sit on every proposition - * ever stored, so without this a whole-graph observation reports them as undeclared relationship - * drift on the very first run against a populated graph, forever. + * Dice's bookkeeping extends past node labels. `HAS_MENTION` and `DERIVED_FROM` sit on every + * proposition ever stored, so without this set a whole-graph observation reports them as undeclared + * relationship drift on every run against a populated graph. * - * These have no per-type shape because the discriminator is the same for all of them and belongs to - * the *other* side: every edge the graph writer projects from domain data carries - * `sourcePropositions`, and no dice bookkeeping edge does. That is the positive marker the - * context-scoped path already selects on, so the whole-graph path uses it too and the two agree. + * These carry no per-type shape because one discriminator covers all of them, and it lives on the + * edge: every edge the graph writer projects from domain data carries `sourcePropositions`, and no + * dice bookkeeping edge carries it. The context-scoped path selects on the same marker, so both + * paths agree. */ val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( "HAS_MENTION", @@ -102,46 +97,42 @@ val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( ) /** - * Drivine / Neo4j implementation of [ObservedSchemaSource]: asks a live graph what it actually - * contains, so a `DeclaredObservedDiffer` can hold it up against what was declared. + * Drivine / Neo4j implementation of [ObservedSchemaSource]: asks a live graph what it contains, so a + * `DeclaredObservedDiffer` can compare it against what was declared. * - * Two genuinely different observation paths, because the database offers no single query that - * answers both: + * There are two observation paths, because the database offers no single query that answers both: * - * - **Whole graph** (`contextId == null`) introspects the database's own catalogue — `db.labels()` - * and `db.relationshipTypes()` — and subtracts dice's bookkeeping from both sides. The - * subtraction is by *shape*, not by name: see [DICE_BOOKKEEPING_LABEL_SHAPES]. A bookkeeping name - * the domain is also using stays in the observation, so an undeclared type can still be reported. - * - **One context** (`contextId != null`) cannot use those procedures at all: they have no notion - * of a context and would answer for the whole database. It derives both sides from that context's - * own data instead: + * - **Whole graph** (`contextId == null`) introspects the database's own catalogue, `db.labels()` + * and `db.relationshipTypes()`, and subtracts dice's bookkeeping from both sides. The subtraction + * goes by node shape; see [DICE_BOOKKEEPING_LABEL_SHAPES]. A bookkeeping name the domain is also + * using stays in the observation, so an undeclared type can still be reported. + * - **One context** (`contextId != null`) cannot use those procedures: they have no notion of a + * context and answer for the whole database. It derives both sides from that context's own data: * - entity types are the distinct `Mention.type` values on that context's propositions; * - relationship types come from the `sourcePropositions` property the graph writer stamps on - * every edge it persists — the ids of the propositions that produced it. An edge belongs to a - * context's set when at least one of those ids names a proposition in that context. That is a - * join on a property, not a tag on the edge, and it deliberately does not collapse: an edge - * sourced from two contexts appears in both. An undeclared relationship type present in your - * context's data is drift in your context regardless of who else also produced it. + * every edge it persists, holding the ids of the propositions that produced it. An edge belongs + * to a context's set when at least one of those ids names a proposition in that context. It is + * a join on that property, and it does not collapse: an edge sourced from two contexts appears + * in both. An undeclared relationship type present in a context's data is drift in that context + * whoever else produced it. * - * The scoped entity side is deliberately *not* filtered at all. Those are Neo4j labels; a mention's + * The scoped entity side is unfiltered. Bookkeeping exclusions are Neo4j labels, while a mention's * `type` is a domain type name an extractor produced, so the two live in different namespaces and - * subtracting one from the other would only ever hide real drift from an app that happens to govern - * a type called `Source`. + * subtracting one from the other would hide real drift from an app governing a type called `Source`. * - * **Two honest limits, neither resolvable from names and shape alone.** First, exclusion is decided - * per *label*, not per node: if any node wearing a bookkeeping label fails dice's shape, the whole - * label stays observed, dice's own nodes included. That direction is chosen on purpose — a - * spuriously reported type is visible and dismissable, a silently hidden one is neither — but it - * does mean a graph mixing a domain `Source` with dice's own will report `Source` every run until - * the domain type is declared. Second, deciding this costs a scan of dice's own labels on every - * unscoped observation (each probe stops at the first non-conforming node, so it is cheap only when - * one exists). Context-scoped checks never pay it, and on a very large graph they are the ones to - * schedule. + * Two limits follow from working off names and shape: + * + * 1. Exclusion is decided per label, not per node. If any node wearing a bookkeeping label fails + * dice's shape, the whole label stays observed, dice's own nodes included, so a graph mixing a + * domain `Source` with dice's own reports `Source` every run until the domain type is declared. + * 2. Deciding it costs a scan of dice's own labels on every unscoped observation. Each probe stops + * at the first non-conforming node, so it is cheap only where one exists. Context-scoped checks + * don't pay it. * * @param persistenceManager Drivine's handle on the `neo` datasource. * @param clock Supplies the snapshot's capture instant. Injectable because that instant ends up in - * a drift report's natural key, so a test that needs two checks to be one observation — or two — - * has to be able to choose it. + * a drift report's natural key, so a test has to be able to choose whether two checks record one + * observation or two. */ open class DrivineObservedSchemaSource( private val persistenceManager: PersistenceManager, @@ -156,8 +147,8 @@ open class DrivineObservedSchemaSource( "CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType" /** - * Bookkeeping labels the domain has also claimed — those carrying at least one node that - * does *not* match dice's shape for them. Whatever this returns stays in the observation. + * Bookkeeping labels the domain has also claimed: those carrying at least one node that + * fails dice's shape for them. Whatever this returns stays in the observation. * * One branch per label, each stopping at the first non-conforming node, unioned into a * single round trip. Label and property names are this file's own compile-time constants; @@ -170,9 +161,9 @@ open class DrivineObservedSchemaSource( } /** - * Bookkeeping relationship types the domain has also claimed — those carrying at least one - * edge with `sourcePropositions`, which is the property the graph writer stamps on every - * edge it projects from domain data and no dice bookkeeping edge has. + * Bookkeeping relationship types the domain has also claimed: those carrying at least one + * edge with `sourcePropositions`, the property the graph writer stamps on every edge it + * projects from domain data. */ private val RELATIONSHIP_TYPES_CLAIMED_BY_DOMAIN: String = """ MATCH ()-[r:${DICE_BOOKKEEPING_RELATIONSHIP_TYPES.joinToString("|")}]->() @@ -197,8 +188,8 @@ open class DrivineObservedSchemaSource( * lives there. * * dice's own edges have no `sourcePropositions` property, so `any(... IN null ...)` is null - * for them and they drop out here rather than needing a name-based exclusion afterwards. - * This is the shape test the whole-graph path mirrors. + * for them and they drop out here, with no name-based exclusion needed afterwards. The + * whole-graph path mirrors this shape test. */ private val RELATIONSHIP_TYPES_IN_CONTEXT = """ MATCH (p:Proposition {contextId: ${'$'}contextId}) @@ -213,9 +204,8 @@ open class DrivineObservedSchemaSource( if (contextId == null) observeWholeGraph() else observeContext(contextId) private fun observeWholeGraph(): ObservedSchema { - // Subtract only the bookkeeping the domain has *not* also claimed. A name dice uses and the - // domain also uses is the domain's for observation purposes, because failing to report an - // undeclared type is the one failure a drift check cannot recover from. + // Subtract only the bookkeeping the domain has not also claimed. A name both dice and the + // domain use counts as the domain's here, so it stays observable as drift. val hiddenLabels = DICE_BOOKKEEPING_LABELS - queryStrings(LABELS_CLAIMED_BY_DOMAIN) val hiddenRelationshipTypes = DICE_BOOKKEEPING_RELATIONSHIP_TYPES - queryStrings(RELATIONSHIP_TYPES_CLAIMED_BY_DOMAIN) @@ -231,9 +221,8 @@ open class DrivineObservedSchemaSource( // Neither side subtracts anything. The scoped queries are already shape-based: mention types // are domain names by construction, and the relationship query selects on // `sourcePropositions`, which only a projected domain edge carries. Subtracting names on top - // would be the bug this class exists to avoid — it would drop a domain relationship type - // that happens to be spelled `DERIVED_FROM` even though the query proved it was the - // domain's. + // would drop a domain relationship type spelled `DERIVED_FROM` that the query had already + // shown to be the domain's. return ObservedSchema( entityTypeNames = queryStrings(MENTION_TYPES_IN_CONTEXT, bindings), relationshipTypeNames = queryStrings(RELATIONSHIP_TYPES_IN_CONTEXT, bindings), diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt index 997c6152..be640bdf 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelRowMappers.kt @@ -225,7 +225,7 @@ private fun deserializeList(serialized: String): List = objectMapper.typeFactory.constructCollectionType(List::class.java, String::class.java) ) -/** Deserialize a JSON string back to a set — what a drift report's drifted type collections are. */ +/** Deserialize a JSON string back to a set, the form a drift report's drifted type fields take. */ private fun deserializeSet(serialized: String): Set = if (serialized.isEmpty()) emptySet() else objectMapper.readValue( @@ -380,8 +380,8 @@ private fun Map<*, *>.str(key: String): String = requireNotNull(this[key]) { "required property '$key' is missing from the stored node" }.toString() /** - * Read a property whose absence is itself meaningful, rather than a fault: a stamp that declared - * no aliases, or a drift report whose check covered the whole graph and so wrote no `contextId`. - * Everything else goes through [str]. + * Read a property whose absence carries meaning: a stamp that declared no aliases, or a drift + * report whose check covered the whole graph and so wrote no `contextId`. Every other property + * goes through [str]. */ private fun Map<*, *>.strOrNull(key: String): String? = this[key]?.toString() diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt index b32ad411..91180e70 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt @@ -21,22 +21,22 @@ import org.drivine.schema.UniquenessConstraintSpec /** * The constraints and node labels the metamodel governance stores need, as plain data. * - * Two things depend on this list and they must not disagree. A host (and the integration-test - * harness) declares [specs] so the stores' MERGEs are race-free; [LABELS] is what - * [DrivineObservedSchemaSource] subtracts from an observation so governance never reports its own - * bookkeeping as domain drift. Keeping both here means adding a governance node label is one edit, - * not two edits in two modules that quietly drift apart. + * Two things depend on this list and have to agree. A host, and the integration-test harness, + * declares [specs] so the stores' MERGEs are race-free; [LABELS] is what + * [DrivineObservedSchemaSource] subtracts from an observation, which keeps governance from reporting + * its own bookkeeping as domain drift. Keeping both here makes adding a governance node label a + * single edit in one module. */ object MetamodelSchema { /** - * Every MERGE these stores perform needs its key to be unique, because a MERGE is only - * race-free when it is. Without that, concurrent saves all miss the match, all create, and the - * history fills with duplicates. + * Every MERGE these stores perform needs its key to be unique, because a MERGE is race-free + * only then. Without that, concurrent saves all miss the match, all create, and the history + * fills with duplicates. * - * The two `sequence` constraints are the safety net under the ordering. They make two records - * of one schema sharing a position impossible to store, so a lost counter update fails loudly - * and retryably instead of quietly making "newest first" arbitrary. + * The two `sequence` constraints back the ordering. They make two records of one schema sharing + * a position impossible to store, so a lost counter update fails with a constraint violation the + * caller can retry. */ fun specs(): List = listOf( // Version stamps -- see DrivineMetamodelVersionStore. @@ -44,9 +44,8 @@ object MetamodelSchema { UniquenessConstraintSpec(label = "MetamodelSchemaCounter", property = "schemaName"), UniquenessConstraintSpec(label = "MetamodelVersion", properties = listOf("schemaName", "sequence")), - // Drift reports -- see DrivineDriftReportStore. The natural key carries `contextKey` rather - // than `contextId` because a Cypher MERGE can't key on a null, so a global report needs a - // non-null stand-in to be as idempotent as a scoped one. + // Drift reports -- see DrivineDriftReportStore. The natural key carries `contextKey` + // because a Cypher MERGE can't key on a null, and a global report has no `contextId`. UniquenessConstraintSpec( label = "MetamodelDriftReport", properties = listOf("schemaName", "versionHash", "capturedAt", "contextKey"), diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt index 336e787c..10c87aa9 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt @@ -46,10 +46,10 @@ import org.springframework.test.context.DynamicPropertySource * [DrivineDriftReportStore] and [DrivinePropositionRepository], with the real differ and the real * quarantine policy. * - * The unit tests in `dice-metamodel` already pin the runner's sequencing against fakes. What only a - * database can answer is whether the three persistent pieces line up: a report written by one store - * must name a hash the *other* store can resolve, and the proposition the policy flagged must come - * back out of the graph flagged. + * The unit tests in `dice-metamodel` pin the runner's sequencing against fakes. What only a database + * can answer is whether the three persistent pieces line up: a report written by one store names a + * hash the other store can resolve, and the proposition the policy flagged comes back out of the + * graph flagged. */ @SpringBootTest(classes = [TestApplication::class]) class DrivineDriftCheckIntegrationTest { @@ -132,14 +132,14 @@ class DrivineDriftCheckIntegrationTest { "a context-scoped check must not show up as a whole-graph one", ) - // 3. Its hash resolves through the *other* store -- the guarantee "stamp before you report" - // exists to buy, and the one that only breaks once both are real. + // 3. Its hash resolves through the version store, which is what stamping before reporting + // guarantees. Only real stores on both sides can show it. val resolved = versionStore.findVersion(schemaName, persisted.single().versionHash) assertNotNull(resolved, "a persisted report named a version hash nothing recorded") assertEquals(declaredVersion, resolved) - // 4. The stranded proposition came back out of the graph flagged, with a reason a person - // can read. + // 4. The stranded proposition came back out of the graph flagged, carrying a readable + // quarantine reason. val reloaded = repository.findById(stranded.id) assertNotNull(reloaded) assertEquals(PropositionStatus.STALE, reloaded!!.status) @@ -169,8 +169,8 @@ class DrivineDriftCheckIntegrationTest { @Test fun `a conforming context reports no drift, and the run is still on the record`() { - // A zero-drift check is a fact worth having, not a no-op -- "we looked and it was clean" is - // the answer an audit needs, and only a persisted report can give it. + // A zero-drift check still persists a report, so an audit can see that the check ran and + // found the context clean. repository.save( Proposition( contextId = contextId, diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt index 97a9d094..04938e41 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftReportStoreIntegrationTest.kt @@ -39,10 +39,10 @@ import java.time.Instant * [DrivineDriftReportStore] against a Neo4j testcontainer. Each test starts from an empty drift log * via [cleanUp]. * - * These are the same questions `DriftReportStoreTest` asks of the in-memory reference, answered in - * Cypher — plus the ones only a database can get wrong: the scope pushed into the query rather than - * applied to a page that has already been cut, a `since` bound that stays exact below the - * millisecond, and the sequence that keeps a limited page from moving between reads. + * These are the questions `DriftReportStoreTest` asks of the in-memory reference, answered in + * Cypher, plus the ones only a database can get wrong: scope pushed into the query ahead of the + * limit, a `since` bound that stays exact below the millisecond, and the sequence that keeps a + * limited page stable between reads. */ @SpringBootTest(classes = [TestApplication::class]) class DrivineDriftReportStoreIntegrationTest { @@ -126,8 +126,8 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `type names carrying delimiter characters survive the round-trip`() { - // The sets are JSON, not a joined string, and these are the characters that would break a - // joined one. Type names come out of LLM extraction and do contain them. + // The sets are stored as JSON, so these characters round-trip. Type names come out of LLM + // extraction and do contain them. val report = DriftReport( schemaName = schemaName, versionHash = "delimiters", @@ -177,10 +177,9 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `since stays exact below the millisecond`() { - // The reason the sort and the bound are stored as (epochSecond, nano) rather than as epoch - // milliseconds. These two are 500 microseconds apart, which truncates to the same - // millisecond: a millis-based bound would sweep the earlier one in and quietly widen the - // window the caller asked for. + // Why the sort and the bound are stored as (epochSecond, nano). These two are 500 + // microseconds apart and truncate to the same millisecond, so a millis-based bound would + // sweep the earlier one in and widen the window the caller asked for. val earlier = epoch.plusNanos(200_000) val later = epoch.plusNanos(700_000) listOf(earlier to "early", later to "late").forEach { (instant, hash) -> @@ -215,10 +214,10 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `reports captured at the same instant hold a stable order across reads`() { - // A global sweep and a context sweep can share a capture instant. The instant alone then - // leaves their order to the database, and with a LIMIT on top the page boundary lands - // somewhere arbitrary -- the same read can return different rows each time. The per-schema - // sequence is what makes the order total, so a page is repeatable. + // A global sweep and a context sweep can share a capture instant. The instant alone leaves + // their order to the database, and under a LIMIT the page boundary lands arbitrarily, so + // the same read can return different rows each time. The per-schema sequence makes the + // order total, which makes a page repeatable. val instant = epoch.plusSeconds(600) val first = DriftReport(schemaName, "tie-1", setOf("A"), emptySet(), instant, contextA) val second = DriftReport(schemaName, "tie-2", setOf("B"), emptySet(), instant, contextB) @@ -247,9 +246,9 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `scoping happens before limiting, not after`() { - // The rule the contract exists to protect. A store that read a limited page and then - // filtered it would answer "no global drift" here: the newest three reports are all - // context-scoped, so the one global report never survives to the filter. + // A store that read a limited page and then filtered it would answer "no global drift" + // here: the newest three reports are all context-scoped, so the one global report never + // reaches the filter. val global = save(1) save(2, contextA) save(3, contextA) @@ -260,8 +259,8 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `a context read pushes its scope down too`() { - // The mirror image: the newest reports are global and another context's, so an in-memory - // filter over a page of 2 would report no drift in context A at all. + // Same shape on the context read: the newest reports are global and another context's, so + // an in-memory filter over a page of 2 would report no drift in context A. val inA = save(1, contextA) save(2) save(3, contextB) @@ -310,9 +309,9 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `a global and a scoped check at the same instant are two records, not an overwrite`() { - // The reason the natural key carries a never-null contextKey. Keying on contextId itself - // would make the global report's key contain a null, which a Cypher MERGE can never match: - // it would take the CREATE branch every time and duplicate on retry. + // Why the natural key carries a never-null contextKey. Keying on contextId itself would put + // a null in the global report's key, which a Cypher MERGE never matches: it would take the + // CREATE branch every time and duplicate on retry. val instant = epoch.plusSeconds(60) val global = DriftReport(schemaName, "same-hash", setOf("Ghost"), emptySet(), instant) val scoped = DriftReport(schemaName, "same-hash", setOf("Ghost"), emptySet(), instant, contextA) @@ -327,12 +326,12 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `a context named after the global marker cannot collide with a global report`() { - // ContextId accepts any non-blank string, so any encoding that stores a bare context id - // alongside a sentinel is one `ContextId(sentinel)` away from a collision: both reports - // would MERGE onto one node and each save would rewrite the other's scope, so a - // context-scoped finding would surface as whole-graph drift. Prefixing every real context - // makes that unrepresentable rather than unlikely -- including for the previous sentinel, - // which a caller may well still be using as a context id. + // ContextId accepts any non-blank string, so an encoding that stores a bare context id + // alongside a sentinel collides as soon as a caller passes `ContextId(sentinel)`: both + // reports MERGE onto one node and each save rewrites the other's scope, surfacing a + // context-scoped finding as whole-graph drift. Prefixing every real context makes the + // collision unrepresentable, including for the sentinel a caller may still be using as a + // context id. val instant = epoch.plusSeconds(60) listOf( ContextId(DriftReportRowMapper.GLOBAL_CONTEXT_KEY), @@ -372,9 +371,9 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `two reports of one schema cannot be stored at the same position`() { - // The safety net under the sequence: whatever the counter does, the database will not hold - // two reports of one schema claiming one place in the tie-break order, so a lost counter - // update is a retryable failure rather than a silently wobbling page. + // The uniqueness constraint backing the sequence: whatever the counter does, the database + // will not hold two reports of one schema at one place in the tie-break order, so a lost + // counter update surfaces as a constraint violation the caller can retry. val first = save(1) assertEquals(1L, storedSequence(first)) @@ -443,9 +442,9 @@ class DrivineDriftReportStoreIntegrationTest { @Test fun `a node with no sort key is excluded in the database, not after the page is cut`() { - // Neo4j sorts null largest, so a node missing the sort key would sort to the front of a DESC - // order, spend one of the caller's limit slots, and then be dropped by the mapper -- hiding - // a perfectly good report behind a broken one. + // Neo4j sorts null largest, so a node missing the sort key would sort to the front of a + // DESC order, spend one of the caller's limit slots, and then be dropped by the mapper, + // hiding a good report behind a broken one. val readable = save(1) persistenceManager.execute( QuerySpecification.withStatement( @@ -468,7 +467,7 @@ class DrivineDriftReportStoreIntegrationTest { // ---- helpers ---- - /** The `sequence` a report node actually holds — storage bookkeeping, so read straight out. */ + /** The `sequence` a report node holds. It is storage bookkeeping, so read it straight out. */ private fun storedSequence(report: DriftReport): Long? = persistenceManager.maybeGetOne( QuerySpecification.withStatement( """ @@ -487,9 +486,8 @@ class DrivineDriftReportStoreIntegrationTest { ) /** - * Run [block] with a listener on the store's logger, and hand back both its result and every - * WARN it emitted. "Skips the row" and "skips the row *and says so*" are different behaviours, - * and only the second is any use to an operator. + * Run [block] with a listener on the store's logger and hand back both its result and every + * WARN it emitted, so a test can assert that a skipped row is also logged. */ private fun capturingStoreWarnings(block: () -> T): Pair> { val logger = LoggerFactory.getLogger(DrivineDriftReportStore::class.java) as Logger diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt index 30d71c86..ec270525 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt @@ -34,10 +34,9 @@ import java.time.Instant /** * [DrivineObservedSchemaSource] against a Neo4j testcontainer. * - * The load-bearing test here is the last group: governance must never observe *itself*. Stamping a - * version and writing a drift report both add nodes to the very graph the next check looks at, so - * without the exclusion every run would report the previous run as drift and the noise would never - * settle. + * The last group covers governance observing its own bookkeeping. Stamping a version and writing a + * drift report both add nodes to the graph the next check looks at, so without the exclusion every + * run reports the previous run as drift. */ @SpringBootTest(classes = [TestApplication::class]) class DrivineObservedSchemaSourceIntegrationTest { @@ -93,8 +92,8 @@ class DrivineObservedSchemaSourceIntegrationTest { @Test fun `a relationship two contexts produced is drift in both of them`() { - // Deliberately does not collapse to one owner: an undeclared relationship type present in - // your context's data is drift in your context, whoever else also produced it. + // The join does not collapse to one owner: an undeclared relationship type present in a + // context's data is drift in that context, whoever else produced it. writeProposition("p-a", tenantA) writeProposition("p-b", tenantB) writeRelationship(type = "SHARED_REL", sourcePropositionIds = listOf("p-a", "p-b")) @@ -152,8 +151,8 @@ class DrivineObservedSchemaSourceIntegrationTest { writeMention("p-a", "m-a", type = "Person") persistenceManager.execute(QuerySpecification.withStatement("CREATE (:Source {key: 'src'})")) persistenceManager.execute(QuerySpecification.withStatement("CREATE (:ProcessedChunk {id: 'chunk'})")) - // Without this the test could pass by observing nothing at all. The database really is - // reporting these labels; the exclusion is what keeps them out of the snapshot. + // Without this precondition the test would pass on an empty observation. It pins that the + // database is reporting these labels, so the exclusion is what keeps them out. assertTrue( rawLabels().containsAll(setOf("Proposition", "Mention", "Source", "ProcessedChunk")), "precondition: the raw catalogue must hold the bookkeeping labels, but was ${rawLabels()}", @@ -169,10 +168,9 @@ class DrivineObservedSchemaSourceIntegrationTest { @Test fun `the metamodel's own nodes are never reported as domain drift`() { - // The self-reference that matters: a drift check stamps a version and writes a report, and - // both land in the graph the *next* check observes. Without the exclusion the second run - // reports MetamodelVersion and MetamodelDriftReport as undeclared entity types, and every - // run after that reports them again. + // A drift check stamps a version and writes a report, and both land in the graph the next + // check observes. Without the exclusion the second run reports MetamodelVersion and + // MetamodelDriftReport as undeclared entity types, as does every run after it. val version = MetamodelVersion("observed-schema", listOf("Person"), emptyMap(), emptyMap(), emptyList()) versionStore.saveVersion(version) reportStore.saveDriftReport( @@ -202,9 +200,9 @@ class DrivineObservedSchemaSourceIntegrationTest { @Test fun `dice's own relationship types are never reported as domain drift`() { - // HAS_MENTION and DERIVED_FROM sit on every proposition ever stored, so this is not an edge - // case: without the exclusion, the first whole-graph check against a populated graph - // reports them as relationship drift, and so does every check after it. + // HAS_MENTION and DERIVED_FROM sit on every proposition ever stored. Without the exclusion, + // the first whole-graph check against a populated graph reports them as relationship drift, + // as does every check after it. writeProposition("p-a", tenantA) writeMention("p-a", "m-a", type = "Person") persistenceManager.execute( @@ -226,13 +224,12 @@ class DrivineObservedSchemaSourceIntegrationTest { ) } - // ---- a name is not a reservation ---- + // ---- Exclusion by shape, not by label name ---- @Test fun `a domain node that only shares a bookkeeping label's name is still observed`() { - // The failure this guards against: reserving the *name* `Source` would mean an app that - // genuinely governs a type called Source could never see it reported as undeclared. A drift - // check that structurally cannot report a type is worse than one that reports too much. + // Excluding the name `Source` would hide an app's own undeclared `Source` type from every + // report. Exclusion goes by node shape so that type stays observable. persistenceManager.execute( QuerySpecification.withStatement("CREATE (:Source {companyName: 'Acme', founded: 1999})"), ) @@ -248,7 +245,7 @@ class DrivineObservedSchemaSourceIntegrationTest { @Test fun `dice's own nodes are still excluded when nothing else claims their label`() { - // The other half: the shape test must not have simply stopped excluding anything. + // The other half of the shape test: dice's own conforming nodes still get excluded. writeProposition("p-a", tenantA) writeMention("p-a", "m-a", type = "Person") persistenceManager.execute(QuerySpecification.withStatement("CREATE (:Source {key: 'src-1'})")) @@ -266,8 +263,8 @@ class DrivineObservedSchemaSourceIntegrationTest { @Test fun `a domain relationship sharing a bookkeeping type's name is still observed`() { - // Same rule on the relationship side, decided by the same marker the scoped path uses: - // an edge carrying sourcePropositions was projected from domain data, whatever it is called. + // Same rule on the relationship side, decided by the marker the scoped path uses: an edge + // carrying sourcePropositions was projected from domain data, whatever it is called. writeProposition("p-a", tenantA) writeRelationship(type = "DERIVED_FROM", sourcePropositionIds = listOf("p-a")) @@ -283,8 +280,8 @@ class DrivineObservedSchemaSourceIntegrationTest { @Test fun `every label the trace and metamodel stores write has a shape entry`() { - // Keeps the shape map one edit away from the schema objects: a new node label added to - // either store without a shape here would silently stop being excluded. + // Keeps the shape map in step with the schema objects: a new node label added to either + // store without a shape here would stop being excluded. (CollectorTraceSchema.LABELS + MetamodelSchema.LABELS).forEach { label -> assertTrue( DICE_BOOKKEEPING_LABEL_SHAPES.containsKey(label), @@ -296,8 +293,8 @@ class DrivineObservedSchemaSourceIntegrationTest { // ---- helpers ---- /** - * A proposition node carrying dice's full shape, because the shape is what tells the observer - * this is dice's node and not a domain one that happens to share the label. + * A proposition node carrying dice's full shape, which is how the observer recognises it as + * dice's own rather than a domain node sharing the label. */ private fun writeProposition(id: String, contextId: ContextId) { persistenceManager.execute( diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt index 56641468..15fc0062 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt @@ -189,9 +189,9 @@ open class TestApplication { ): DrivineDriftReportStore = DrivineDriftReportStore(persistenceManager) /** - * On the system clock, not [PinnableClock]. An observation's capture instant is half of a drift - * report's identity, so two checks sharing one really are the same record — pinning here by - * default would silently collapse consecutive checks into a single report. + * Left on the system clock. An observation's capture instant is half of a drift report's + * identity, so two checks sharing one instant are the same record; pinning [PinnableClock] here + * by default would collapse consecutive checks into a single report. */ @Bean open fun observedSchemaSource( diff --git a/docs/design/architecture.md b/docs/design/architecture.md index bd7e3a37..aa92be6c 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -12,7 +12,7 @@ DICE is a multi-module Maven build. Each module's intent, and what it's allowed | Module | Intent | |---|---| | `dice` | The core: proposition model, pipeline, gates, projection interfaces, query facades, agent tools, REST controllers. In-memory implementations only — no database driver. | -| `dice-storage` | The durable Neo4j backend: `Drivine`-based repository, graph/Prolog/lineage projectors, schema and index bootstrap, and the whole governance persistence side — `MetamodelVersionStore`, the `DriftReportStore` drift log, and the `ObservedSchemaSource` that asks the live graph what it actually holds (excluding dice's own bookkeeping labels and edges, so governance never observes itself). Depends on `dice` and `dice-metamodel`. | +| `dice-storage` | The durable Neo4j backend: `Drivine`-based repository, graph/Prolog/lineage projectors, schema and index bootstrap, and the governance persistence side: `MetamodelVersionStore`, the `DriftReportStore` drift log, and the `ObservedSchemaSource` that asks the live graph what it holds, excluding dice's own bookkeeping labels and edges so governance doesn't observe itself. Depends on `dice` and `dice-metamodel`. | | `dice-storage-autoconfigure` | Spring Boot autoconfiguration that wires `dice-storage`'s beans (repository, projectors, trust scorer) into a host application. Depends on `dice-storage`. | | `dice-ingestion` | Content-hash dedup ledger and source adapters that sit in front of `PropositionPipeline`, so the same artifact is never extracted twice concurrently. Depends on `dice`. | | `dice-report` | Rationale and structured report generation over propositions and their lineage. Depends on `dice`. | diff --git a/docs/design/metamodel-drift.md b/docs/design/metamodel-drift.md index 3cd47f96..8857cf6f 100644 --- a/docs/design/metamodel-drift.md +++ b/docs/design/metamodel-drift.md @@ -619,41 +619,36 @@ matters. Both storage-side contracts are implemented in `dice-storage`, against Neo4j via Drivine. `DrivineDriftReportStore` keeps each check as a `(:MetamodelDriftReport)` node, MERGEd on the -natural key `(schemaName, versionHash, capturedAt, contextKey)` — where `contextKey` is `global` for +natural key `(schemaName, versionHash, capturedAt, contextKey)`, where `contextKey` is `global` for an unscoped check and `ctx:` for a scoped one, because a Cypher MERGE cannot key on a null. The -prefix is what makes the encoding injective: `ContextId` accepts any non-blank string, so an -unprefixed id plus a bare sentinel would let a context named after the sentinel share a key with the -global bucket and silently rewrite its scope. Each of the three bounded reads is -its own statement with its scope in the `WHERE` clause and the `LIMIT` after it, which is the whole -point: filtering a page that has already been cut applies the limit before the scope and can report -zero global drift while plenty sits in the store. Reports come back newest first by capture instant, -compared to the nanosecond so a `since` window stays exact, with a per-schema counter breaking exact -ties so a limited page doesn't move between reads. +prefix makes the encoding injective: `ContextId` accepts any non-blank string, so an unprefixed id +plus a bare sentinel would let a context named after the sentinel share a key with the global bucket +and rewrite its scope. Each of the three bounded reads is its own statement with its scope in the +`WHERE` clause and the `LIMIT` after it. Filtering a page that has already been cut applies the limit +ahead of the scope, and can report zero global drift while plenty sits in the store. Reports come +back newest first by capture instant, compared to the nanosecond so a `since` window stays exact, +with a per-schema counter breaking exact ties so a limited page is repeatable. `DrivineObservedSchemaSource` takes the snapshot. Unscoped, it reads the database's own catalogue (`db.labels()`, `db.relationshipTypes()`); scoped to a context, it derives entity types from that context's mentions and relationship types from the `sourcePropositions` each projected edge carries. -Either way it subtracts dice's own bookkeeping — the proposition, mention, provenance, lineage, +Either way it subtracts dice's own bookkeeping: the proposition, mention, provenance, lineage, collector-trace and metamodel node labels, and the `HAS_MENTION`/`DERIVED_FROM`-style edges. That -subtraction is load-bearing rather than tidy: stamping a version and writing a report both add nodes -to the very graph the next check looks at, so without it every run would report the previous run as -drift and the noise would never settle. - -**The subtraction is by shape, not by name.** Reserving the name `Source` would mean an app that -genuinely governs a type called `Source` could never see it reported as undeclared — a drift check -that structurally cannot report a type is worse than one that reports too much. So a bookkeeping -label is excluded only while every node carrying it matches dice's shape for it (dice's `Source` -nodes carry `key`, its governance nodes carry `schemaName`, and so on), and a bookkeeping -relationship type only while no edge of that type carries `sourcePropositions` — the marker the -graph writer stamps on every edge it projects from domain data, and the same one the context-scoped -query already selects on. - -Two limits are worth stating rather than papering over. Exclusion is decided per label, not per -node, so a graph mixing a domain `Source` with dice's own reports `Source` every run until the type -is declared — deliberately the visible direction to fail in. And deciding it costs a scan of dice's -own labels on each unscoped observation; context-scoped checks never pay it. - -Hosts declare the constraints these stores need (see `MetamodelSchema`); a MERGE is only race-free +subtraction is load-bearing. Stamping a version and writing a report both add nodes to the graph the +next check looks at, so without it every run reports the previous run as drift. + +The subtraction goes by node shape, so that a domain type called `Source` stays visible. A +bookkeeping label is excluded only while every node carrying it matches dice's shape for it (dice's +`Source` nodes carry `key`, its governance nodes carry `schemaName`, and so on), and a bookkeeping +relationship type only while no edge of that type carries `sourcePropositions`, the marker the graph +writer stamps on every edge it projects from domain data. The context-scoped query selects on the +same marker. + +Two limits follow. Exclusion is decided per label, not per node, so a graph mixing a domain `Source` +with dice's own reports `Source` every run until the type is declared. And deciding it costs a scan +of dice's own labels on each unscoped observation; context-scoped checks don't pay it. + +Hosts declare the constraints these stores need (see `MetamodelSchema`); a MERGE is race-free only under a uniqueness constraint on the key it merges on. ## What comes next From d16abf1ca82bb543ae86274806737d0368e64612 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:18:40 -0400 Subject: [PATCH 03/10] Observe the graph from one snapshot, and separate mention types from labels The whole observation now comes from a single read transaction, so a concurrent write landing between two schema queries can no longer produce a combined observation the graph never held. Neo4j reads committed data per row, so the KDoc states what that leaves open rather than claiming snapshot isolation. ObservedSchema gains an entityTypeBasis. A mention's type is domain data an extractor wrote, living in its own namespace apart from graph labels, so a MENTION_TYPES observation compares against declared type names and aliases with no widening to inherited labels. Before this, a mention typed Agent conformed whenever Agent was a parent label of a governed Person, and an undeclared type rode into the graph on that. GRAPH_LABELS keeps the previous semantics and stays the default, and @JvmOverloads preserves the old three-argument constructor for Java callers. DrivineMetamodelVersionStore now overrides sweptVersion and markSwept, tracking sweptContentHash on the schema's counter node, so the durable store follows the baseline independently of write order. Enabling transaction management is the larger change here. dice-storage's test harness never enabled it, and neither did production: Spring Boot 4.1 moved TransactionAutoConfiguration into spring-boot-transaction, which nothing on the classpath brought in. Every @Transactional in the module was decorative. Adding the module dependency makes them real, and Boot's configuration backs off for a consumer who already enables it. That activation exposed a latent assumption. DrivinePropositionRepository documents holding its stripe lock across the commit, which only held while its TransactionTemplate ran its own transaction; an ambient transaction made it join instead, moving the commit outside the lock. REQUIRES_NEW restores it, pinned by a test that pauses the writer inside beforeCommit and admits a sibling only there. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- CHANGELOG.md | 87 +++++ .../embabel/dice/metamodel/MetamodelDiff.kt | 18 +- .../embabel/dice/metamodel/ObservedSchema.kt | 43 ++- .../support/StructuralMetamodelDiffer.kt | 46 ++- .../dice/metamodel/MetamodelDifferTest.kt | 49 +++ dice-storage-autoconfigure/pom.xml | 13 + ...StorageTransactionAutoConfigurationTest.kt | 116 +++++++ .../storage/DrivineMetamodelVersionStore.kt | 69 ++++ .../storage/DrivineObservedSchemaSource.kt | 48 +++ .../storage/DrivinePropositionRepository.kt | 23 +- ...stractMetamodelVersionStoreContractTest.kt | 84 +++++ .../DrivineDriftCheckIntegrationTest.kt | 7 +- ...vineObservedSchemaSourceIntegrationTest.kt | 13 + ...dSchemaSourceTransactionIntegrationTest.kt | 114 ++++++ ...itionRepositoryDedupRaceIntegrationTest.kt | 324 ++++++++++++++++++ .../DrivinePropositionStoreIntegrationTest.kt | 52 +++ .../embabel/dice/storage/TestApplication.kt | 1 + docs/design/metamodel-drift.md | 24 ++ docs/design/metamodel-versioning.md | 9 +- 19 files changed, 1109 insertions(+), 31 deletions(-) create mode 100644 dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageTransactionAutoConfigurationTest.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceTransactionIntegrationTest.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionRepositoryDedupRaceIntegrationTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 5804d8c0..314a7aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -371,6 +371,93 @@ and the consumer PRs that deliver it). swap in `MetamodelSchema.specs()`, a superset. The drift-report store needs three more: `MetamodelDriftReport(schemaName, versionHash, capturedAt, contextKey)`, `MetamodelDriftReportCounter(schemaName)`, and `MetamodelDriftReport(schemaName, sequence)`. +- Three corrections to the Drivine drift-persistence slice above, closing gaps a review found before + the pieces ever reached a released build. + `DrivineObservedSchemaSource.observe` is now `@Transactional(readOnly = true)`, so the several + queries a whole-graph or context-scoped observation issues run inside one Neo4j transaction and are + assembled from it, the pattern `DrivineCollectorTraceStore.findEdgesByRun` already uses for the same + reason. A concurrent graph write landing between separately-transacted queries could previously + combine into an `ObservedSchema` describing a graph state that never existed at any single instant. + This narrows the exposure to Neo4j's own per-transaction read-committed semantics — a write that + commits while the transaction is still open can still reach a later statement inside it. The + method's own KDoc states that residual honestly; Neo4j offers no full snapshot isolation to claim. + Second, `ObservedSchema` gains `entityTypeBasis: EntityTypeBasis` (`GRAPH_LABELS` default, + `MENTION_TYPES`), stating what kind of name `entityTypeNames` holds. The shared differ was comparing + a context-scoped observation's `Mention.type` values against the same declared-labels set a + whole-graph observation's Neo4j labels compare against, so a mention typed `Agent` passed drift + detection when `Agent` was only a parent label of governed `Person` and nothing declared `Agent` a + type of its own — an inherited-label escape hatch for undeclared mention types. `DrivineObservedSchemaSource` + tags its context-scoped observation `MENTION_TYPES`; `StructuralMetamodelDiffer.diffAgainstObserved` + now compares a `MENTION_TYPES` observation against declared type names and their declared former + names only, with no widening to inherited labels. Third, `DrivineMetamodelVersionStore` overrides + `sweptVersion`/`markSwept`, tracking the reconciled baseline as a `sweptContentHash` property on the + schema's own `(:MetamodelSchemaCounter)` node, moved only by `markSwept` and left untouched by an + ordinary `saveVersion`, the same independence `InMemoryMetamodelVersionStore` already had. Without + this override the durable store inherited the interface's forwarding default, and every run's own + history-stamping write — dry, scoped, or crashed alike — silently consumed the very signal + `DefaultDriftCheckRunner`'s declared-vs-previous comparison depends on, a gap the drift-runner slice + above called out and deferred to this one. + **Compatibility: additive.** `ObservedSchema` gains a defaulted constructor parameter under + `@JvmOverloads`, so the pre-existing three-argument constructor survives in the compiled class + alongside the new four-argument one, confirmed by running `javap` on the compiled class after + compiling. Every existing Kotlin or Java caller and canned test fixture keeps compiling, keeping + its prior (`GRAPH_LABELS`) reading. `DrivineMetamodelVersionStore` and `DrivineObservedSchemaSource` + gain behavior on existing methods; no signature changed. `AbstractMetamodelVersionStoreContractTest` + gains four `sweptVersion`/`markSwept` cases; the graph store now passes all four, and three of + them — the null-until-swept case, the independence-from-a-later-new-stamp case, and the + independence-from-a-later-re-save case — previously failed against the forwarding default. + +- `dice-storage-autoconfigure` now depends on `spring-boot-transaction`. On Spring Boot 4, a + `PlatformTransactionManager` bean alone does not activate `@Transactional`: the interceptor that + reads the annotation lives in that separate module, which was missing here. Every `@Transactional` + across `dice-storage` — around 78 of them — was silently inert in any application built on this + autoconfiguration module, running with no transactional guarantees at all despite the annotations + reading as if it did. `TransactionAutoConfiguration`'s own `@ConditionalOnMissingBean` on + `AbstractTransactionManagementConfiguration` means it backs off cleanly for a consumer that already + enables transaction management itself, so this addition is safe to double up on. + **Compatibility: behavioral.** This is a genuine runtime change on upgrade: `@Transactional` + methods across `dice-storage` start actually running inside transactions for the first time in + any consumer using this autoconfiguration. `DrivinePropositionRepository.save` + is direct proof that activation can expose a latent assumption written against the inert state: its + dedup path ran a `TransactionTemplate` under an ambient (but previously inert) class-level + `@Transactional`, with the stripe lock documented as held across the template's own commit. With + transaction management genuinely active, the template's default propagation joined the now-real + ambient transaction and deferred that commit past the lock release, reopening the exact race the + KDoc claimed could not happen, and leaving the constraint-violation recovery path one participation + away from `UnexpectedRollbackException`. Fixed by giving that `TransactionTemplate` + `Propagation.REQUIRES_NEW`, so it always commits independently of whatever transaction is already + open, proven with a test that pins a sibling save into the exact window between the writer's stripe + lock release and its commit, forcing the overlap deterministically so nothing depends on scheduling + luck: it fails against the joined-transaction behavior and passes with the independent one, every + run. + + **Upgrade guidance for consumers of `dice-storage-autoconfigure`:** + - Audit your own `@Transactional` usage too, alongside dice's own. Any `@Transactional` method in + your application that quietly relied on nothing actually enforcing it starts running for real the + moment this dependency lands on your classpath. + - Do not wrap `GraphDecayManager.materialize`/`materializeAll` in your own `@Transactional` + boundary. Its KDoc already warned against this, because the sweep's `CALL { ... } IN + TRANSACTIONS` batching depends on running in its own implicit transaction; that warning had no + teeth while `@Transactional` was inert, and an enclosing transaction now makes the batched + Cypher fail outright. + - `DrivinePropositionRepository.reembedAll()` now genuinely holds one Neo4j connection and + transaction open for its entire run, including every call out to your `EmbeddingService`. If + that service is remote or slow, budget for a database connection held that whole time. Write + locks are a separate, narrower concern: the batch write only starts in `executeBatch`, after + every embedding has already been computed, so lock contention with concurrent writers is + confined to that last stretch near the end of the run. + - `@Transactional(readOnly = true)` is worth knowing the limits of on this Drivine version + (0.0.79), confirmed by reading the resolved jar's bytecode: `isReadOnly()` feeds a debug log + line and nothing else, so a write reached through one of dice's read-only-annotated methods is + still permitted — that part is unchanged. What genuinely does change is a consequence of the + surrounding `@Transactional` advice becoming real: commit grouping and rollback. Such a write + now lands inside the same real transaction as everything else that method does and commits + together with the rest of the call, and a later rollback-triggering exception in that same call + now rolls it back too. Previously, with no active transaction wrapping it, the write had already + committed independently and stayed committed whatever happened next. Which exceptions trigger + that rollback follows Spring's defaults: a `RuntimeException` or an `Error` rolls back and a + checked exception commits, and custom rollback rules can override either behaviour. + - Optional source revisions in the `dice` core provenance model, the first slice of DICE #64. `ProvenanceEntry` gains a sixth field, `sourceRevision`: an opaque, provider-defined string, non-blank when present, recording which version of a source a claim was read from. diff --git a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/MetamodelDiff.kt b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/MetamodelDiff.kt index ec9932b9..e7e92d8b 100644 --- a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/MetamodelDiff.kt +++ b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/MetamodelDiff.kt @@ -611,12 +611,18 @@ class MetamodelDiff( * about whether a declared property's shape matches what the graph stores. Property signatures are * compared where both sides have them: declared against declared, in [MetamodelDiff]. * - * **A declared label counts as declared.** A graph reports labels, and a type usually carries more - * than one: declaring `Person` with parent `Agent` puts both labels on every `Person` node. So the - * declared side of the drift comparison is every entity type name *plus* every label those types - * declare. Without the labels, an inherited label would be reported as undeclared drift on a schema - * nobody had changed. [unobservedEntityTypes] stays on the type names alone, because "declared but - * with no data" is a statement about types, and a parent label was never a type in its own right. + * **What counts as declared depends on what kind of name was observed**, per + * [ObservedSchema.EntityTypeBasis]. A graph reports labels, and a type usually carries more than + * one: declaring `Person` with parent `Agent` puts both labels on every `Person` node, so against a + * [ObservedSchema.EntityTypeBasis.GRAPH_LABELS] observation the declared side is every entity type + * name *plus* every label those types declare — without the labels, an inherited label would be + * reported as undeclared drift on a schema nobody had changed. A mention's `type` is domain data an + * extractor wrote, living in its own namespace apart from graph labels, so against a [ObservedSchema.EntityTypeBasis.MENTION_TYPES] + * observation the declared side stays on entity type names, with no widening to inherited labels: a + * mention typed `Agent` conforms only when `Agent` is itself a declared type, whatever labels a + * governed `Person` carries. [unobservedEntityTypes] stays on the type names alone either way, + * because "declared but with no data" is a statement about types, and a parent label was never a + * type in its own right. * * @property declared The schema as declared at snapshot time, stamp and bare relationship names. * @property observedSchema What the live graph held at snapshot time. diff --git a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt index 7526a917..24dbcbfc 100644 --- a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt +++ b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt @@ -36,17 +36,49 @@ import java.util.Objects * contributing types that come and go, so what is in the graph can drift from what was declared, * and this module needs a way to talk about that without depending on a particular graph driver. * - * @property entityTypeNames Entity type (label) names observed in the graph. + * @property entityTypeNames Entity type names observed. What counts as a name depends on + * [entityTypeBasis]: a Neo4j graph label, or a mention's own `type` field. * @property relationshipTypeNames Relationship type names observed in the graph. * @property capturedAt When this snapshot was taken. + * @property entityTypeBasis What kind of name [entityTypeNames] holds, which decides what the + * declared side of a comparison has to be. See [EntityTypeBasis]. */ @ApiStatus.Experimental -class ObservedSchema( +class ObservedSchema @JvmOverloads constructor( entityTypeNames: Set, relationshipTypeNames: Set, val capturedAt: Instant, + val entityTypeBasis: EntityTypeBasis = EntityTypeBasis.GRAPH_LABELS, ) { + /** + * What kind of name [ObservedSchema.entityTypeNames] holds. A [DeclaredObservedDiffer] needs to + * know this, because the two kinds of name compare against different declared sets. + */ + enum class EntityTypeBasis { + + /** + * A Neo4j label, read off `db.labels()` on a whole-graph observation. A node carries every + * label in its declared type's hierarchy, so declaring `Person` with parent `Agent` puts both + * labels on every `Person` node. The declared side of a comparison against this basis has to + * include every label the declaration's types carry, on top of the type names themselves, or + * an inherited parent label reads as undeclared drift on a schema nobody touched. + */ + GRAPH_LABELS, + + /** + * The `type` a mention was extracted as, read off `Mention.type` on a context-scoped + * observation. This is domain data an extractor wrote, living in its own namespace apart from graph labels, and a graph's + * label hierarchy has no bearing on it: a mention typed `Agent` claimed to be an `Agent`, and + * that claim stands or falls on whether `Agent` is itself a declared type, whatever labels a + * governed `Person` happens to carry. The declared side of a comparison against this basis + * stays on declared type names (plus their declared former names — old data can still carry a + * type's pre-rename spelling); it must not widen to include inherited labels, or an undeclared + * mention type escapes detection by riding a governed type's parent label. + */ + MENTION_TYPES, + } + // Both sets are copied into immutable ones, keeping the order they arrived in. A snapshot // describes one moment and must not change afterwards, and a backend typically builds these from // a mutable set it fills as it walks query results. Kotlin's read-only `Set` is a compile-time @@ -61,13 +93,14 @@ class ObservedSchema( other is ObservedSchema && entityTypeNames == other.entityTypeNames && relationshipTypeNames == other.relationshipTypeNames && - capturedAt == other.capturedAt + capturedAt == other.capturedAt && + entityTypeBasis == other.entityTypeBasis - override fun hashCode(): Int = Objects.hash(entityTypeNames, relationshipTypeNames, capturedAt) + override fun hashCode(): Int = Objects.hash(entityTypeNames, relationshipTypeNames, capturedAt, entityTypeBasis) override fun toString(): String = "ObservedSchema(entityTypeNames=$entityTypeNames, relationshipTypeNames=$relationshipTypeNames, " + - "capturedAt=$capturedAt)" + "capturedAt=$capturedAt, entityTypeBasis=$entityTypeBasis)" private companion object { diff --git a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt index b6c80177..b4c88211 100644 --- a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt +++ b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt @@ -138,35 +138,49 @@ class StructuralMetamodelDiffer : MetamodelDiffer, DeclaredObservedDiffer { val declaredTypes = declared.version.entityTypeNames.toSet() val observedTypes = observed.entityTypeNames - // What a graph reports is labels, and a type carries every label in its hierarchy: declare - // `Person` with parent `Agent` and every Person node comes back carrying both. Comparing - // observed labels against type names alone would call `Agent` undeclared drift on a schema - // nobody had touched, so the declared side of the drift check is the type names plus every - // label those types declare. + // Declared former names count as declared on both bases below. Nodes written before a type + // was renamed keep the old label, and a mention extracted before the rename keeps the old + // type name, and either way the rename was declared, so the old name is known. Leaving it out + // would report a declared rename as drift on every check from then on. // // A declared name can also be fully qualified where the observed one is simple. The stamp // holds `com.example.Person` for a JVM-backed type, extraction records the mention as - // `Person`, and the graph reports `Person`. Both spellings of every declared type go on the - // declared side, through DeclaredSchema.entityTypeOwnLabels. - // - // Declared former names count as declared too. Nodes written before a type was renamed keep - // the old label, and the rename was declared, so the old label is known. Leaving it out - // would report a declared rename as drift on every check from then on. A former name is a - // declared name, so it brings its own label with it the same way. + // `Person`, and the graph reports `Person`. Both spellings of every declared type — and of + // every declared former name — go on the declared side, through + // DeclaredSchema.entityTypeOwnLabels and ownLabelsOf. val declaredAliases = declared.version.entityTypeAliases.values.flatten() - val declaredLabels = declaredTypes + + val declaredEitherSpelling = declaredTypes + declared.entityTypeOwnLabels + - declared.version.entityTypeLabels.values.flatten() + declaredAliases + ownLabelsOf(declaredAliases) + // What counts as "declared" on the observed side depends on what kind of name is being + // compared; see ObservedSchema.EntityTypeBasis. + val declaredEntityTypeBasis = when (observed.entityTypeBasis) { + // A graph reports labels, and a type carries every label in its hierarchy: declare + // `Person` with parent `Agent` and every Person node comes back carrying both. Comparing + // observed labels against type names alone would call `Agent` undeclared drift on a + // schema nobody had touched, so the declared side of the drift check is the type names + // plus every label those types declare. + ObservedSchema.EntityTypeBasis.GRAPH_LABELS -> + declaredEitherSpelling + declared.version.entityTypeLabels.values.flatten() + // A mention's `type` is domain data an extractor wrote, living in its own namespace + // apart from graph labels, so a + // governed type's inherited parent label has no bearing on it: a mention typed `Agent` + // only conforms when `Agent` is itself a declared type or former name, whatever labels a + // governed `Person` carries. Widening this to declared labels would let an undeclared + // mention type escape detection by riding a governed type's parent label. + ObservedSchema.EntityTypeBasis.MENTION_TYPES -> + declaredEitherSpelling + } + // A type the host's dictionary names but the selector leaves outside governance is a known // type, and the drift check has to recognise it as such. It gets its own excluded set, - // separate from declaredLabels, because it must stay out of unobservedEntityTypes below: a + // separate from the declared basis, because it must stay out of unobservedEntityTypes below: a // governance-exempt type with no data isn't the informational case that bucket describes. // These names come off the same dictionary the governed ones do, so they can be fully // qualified in the same way, and their own labels are excluded alongside them. - val excludedFromDrift = declaredLabels + + val excludedFromDrift = declaredEntityTypeBasis + declared.ungovernedEntityTypeNames + ownLabelsOf(declared.ungovernedEntityTypeNames) diff --git a/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt b/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt index bd44c1cd..e8a36d33 100644 --- a/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt +++ b/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt @@ -507,6 +507,17 @@ class MetamodelDifferTest { capturedAt = Instant.parse("2026-01-01T00:00:00Z"), ) + /** An observation tagged as mention types, on the other basis — see [ObservedSchema.EntityTypeBasis]. */ + private fun observedMentionTypes( + entityTypeNames: Set, + relationshipTypeNames: Set = emptySet(), + ): ObservedSchema = ObservedSchema( + entityTypeNames = entityTypeNames, + relationshipTypeNames = relationshipTypeNames, + capturedAt = Instant.parse("2026-01-01T00:00:00Z"), + entityTypeBasis = ObservedSchema.EntityTypeBasis.MENTION_TYPES, + ) + @Test fun `an observed type with no declaration is reported as drift`() { val diff = declaredObservedDiffer.diffAgainstObserved( @@ -752,6 +763,44 @@ class MetamodelDifferTest { assertTrue(diff.driftedEntityTypes.isEmpty()) } + /** + * The escape this closes: `Mention.type` is domain data an extractor wrote, living in its own + * namespace apart from graph labels, so a governed type's inherited parent label must not let + * an undeclared mention type pass. `Agent` is only a parent label of governed `Person` here — nothing declares `Agent` + * as a type in its own right — so a mention typed `Agent` has to read as drift. + */ + @Test + fun `a mention typed with a parent label of a governed type is drift`() { + val personIsAnAgent = DeclaredSchema.from( + DataDictionary.fromDomainTypes( + "test", + listOf(DynamicType(name = "Person", parents = listOf(DynamicType(name = "Agent")))), + ), + ) + val diff = declaredObservedDiffer.diffAgainstObserved( + personIsAnAgent, + observedMentionTypes(entityTypeNames = setOf("Agent")), + ) + assertTrue(diff.hasDrift, "an undeclared mention type must not escape through an inherited label") + assertEquals(setOf("Agent"), diff.driftedEntityTypes) + } + + /** The same mention type against a graph-label observation is the pre-existing, correct case. */ + @Test + fun `the same inherited label reported as a graph label is not drift`() { + val personIsAnAgent = DeclaredSchema.from( + DataDictionary.fromDomainTypes( + "test", + listOf(DynamicType(name = "Person", parents = listOf(DynamicType(name = "Agent")))), + ), + ) + val diff = declaredObservedDiffer.diffAgainstObserved( + personIsAnAgent, + observed(entityTypeNames = setOf("Agent")), + ) + assertFalse(diff.hasDrift, "a graph label observation still counts an inherited label as declared") + } + @Test fun `a label matching no declared type or label is still drift`() { val personIsAnAgent = DeclaredSchema.from( diff --git a/dice-storage-autoconfigure/pom.xml b/dice-storage-autoconfigure/pom.xml index 85a85d1c..6ad1e6c7 100644 --- a/dice-storage-autoconfigure/pom.xml +++ b/dice-storage-autoconfigure/pom.xml @@ -40,6 +40,19 @@ org.springframework.boot spring-boot-autoconfigure + + + org.springframework.boot + spring-boot-transaction + org.springframework.boot spring-boot-configuration-processor diff --git a/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageTransactionAutoConfigurationTest.kt b/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageTransactionAutoConfigurationTest.kt new file mode 100644 index 00000000..691268d6 --- /dev/null +++ b/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageTransactionAutoConfigurationTest.kt @@ -0,0 +1,116 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage.autoconfigure + +import com.embabel.agent.api.common.Ai +import com.embabel.common.ai.model.EmbeddingService +import com.embabel.dice.proposition.PropositionRepository +import org.assertj.core.api.Assertions.assertThat +import org.drivine.manager.GraphObjectManager +import org.drivine.manager.PersistenceManager +import org.junit.jupiter.api.Test +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.springframework.aop.framework.Advised +import org.springframework.aop.support.AopUtils +import org.springframework.beans.factory.getBean +import org.springframework.boot.autoconfigure.AutoConfigurations +import org.springframework.boot.test.context.runner.ApplicationContextRunner +import org.springframework.boot.transaction.autoconfigure.TransactionAutoConfiguration +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.interceptor.TransactionInterceptor + +/** + * Proves that a real consumer of `dice-storage-autoconfigure` gets working `@Transactional` advice + * on the graph-backed [PropositionRepository] bean, going beyond confirming the bean merely exists. + * + * A `PlatformTransactionManager` bean alone is not enough: Spring only wires the interceptor that + * reads `@Transactional` when transaction management is actually enabled somewhere. Boot 4 moved + * that wiring into the separate `spring-boot-transaction` module. Both tests below run the actual + * bean through the actual autoconfiguration stack and inspect the resulting object's proxy + * advisors, so a regression here (e.g. dropping the `spring-boot-transaction` dependency again) + * fails a concrete assertion, catching it even if the annotation itself silently stays inert. + */ +class DiceStorageTransactionAutoConfigurationTest { + + private val baseRunner = ApplicationContextRunner() + .withUserConfiguration(GraphBackendStubConfig::class.java) + .withPropertyValues("embabel.dice.store.type=graph") + + @Test + fun `with spring-boot-transaction autoconfiguration, the proposition store bean is a real transactional proxy`() { + baseRunner + .withConfiguration( + AutoConfigurations.of( + DiceStorageAutoConfiguration::class.java, + TransactionAutoConfiguration::class.java, + ) + ) + .run { ctx -> + val repository = ctx.getBean() + + assertThat(AopUtils.isAopProxy(repository)) + .withFailMessage("expected the proposition store bean to be an AOP proxy once transaction " + + "management is enabled, but it was a plain %s", repository::class.java) + .isTrue() + + val advisors = (repository as Advised).advisors.toList() + assertThat(advisors.any { it.advice is TransactionInterceptor }) + .withFailMessage("expected a TransactionInterceptor advisor on the proxy, found: %s", advisors) + .isTrue() + } + } + + /** + * Same wiring with [TransactionAutoConfiguration] left out, standing in for what every consumer + * got before `spring-boot-transaction` was added as a dependency: a plain, unproxied bean. This + * is what pinned P1-b down in the first place — the `PlatformTransactionManager` bean existing + * was never sufficient on its own. + */ + @Test + fun `without spring-boot-transaction autoconfiguration, the proposition store bean is not proxied`() { + baseRunner + .withConfiguration(AutoConfigurations.of(DiceStorageAutoConfiguration::class.java)) + .run { ctx -> + val repository = ctx.getBean() + assertThat(AopUtils.isAopProxy(repository)).isFalse() + } + } + + @Configuration(proxyBeanMethods = false) + class GraphBackendStubConfig { + @Bean + fun ai(): Ai { + val ai = mock() + val embeddingService = mock() + whenever(embeddingService.dimensions).thenReturn(1536) + whenever(embeddingService.name).thenReturn("stub-embedding-model") + whenever(ai.withDefaultEmbeddingService()).thenReturn(embeddingService) + return ai + } + + @Bean + fun graphObjectManager(): GraphObjectManager = mock() + + @Bean + fun persistenceManager(): PersistenceManager = mock() + + @Bean + fun platformTransactionManager(): PlatformTransactionManager = mock() + } +} diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt index 02f7105e..240e9851 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt @@ -67,6 +67,15 @@ import java.time.Clock * re-stamps its schema on every pass anyway, so for that caller the next pass already is the * retry. * + * The reconciled baseline [sweptVersion] answers is tracked apart from that write-order history, the + * same way [InMemoryMetamodelVersionStore][com.embabel.dice.metamodel.InMemoryMetamodelVersionStore] + * keeps a separate `swept` map: as `sweptContentHash`, a property on the schema's own + * `(:MetamodelSchemaCounter)` node, moved only by [markSwept]. Nothing about an ordinary [saveVersion] + * touches it, which is what makes a dry run, a scoped run, or a crash mid-sweep leave the baseline + * exactly where it was — see `MetamodelVersionStore.sweptVersion`'s own doc for why a store that + * doesn't track this independently answers `latestVersion` instead, and gets the wrong answer once a + * declaration cycles back to a stamp it already used. + * * @param persistenceManager Drivine's handle on the `neo` datasource. * @param clock supplies the instant a version is stamped as saved at. Injectable so a test can pin * the instants of two saves. @@ -139,6 +148,30 @@ class DrivineMetamodelVersionStore( RETURN n ORDER BY coalesce(n.sequence, -1) DESC """.trimIndent() + + /** + * Move the reconciled baseline. MERGEs the schema's counter node in case a schema's very + * first save is also its first sweep, though the ordinary [saveVersion] call [markSwept] + * makes first will normally have already created it. + */ + private val MARK_SWEPT = """ + MERGE (c:MetamodelSchemaCounter {schemaName: ${'$'}schemaName}) + SET c.sweptContentHash = ${'$'}contentHash + """.trimIndent() + + /** + * The reconciled baseline's content hash, or no row at all when the schema has never been + * swept. Neo4j never stores an explicit null property, so `sweptContentHash IS NOT NULL` + * reads as "the counter node exists and carries this property" — true only once [markSwept] + * has run for the schema. The Cypher `WHERE` filter answers this in the database itself, so a + * schema with no counter node at all and one whose counter exists but has never been swept + * both come back the same way: no row. + */ + private val SWEPT_CONTENT_HASH = """ + MATCH (c:MetamodelSchemaCounter {schemaName: ${'$'}schemaName}) + WHERE c.sweptContentHash IS NOT NULL + RETURN c.sweptContentHash AS sweptContentHash + """.trimIndent() } override fun saveVersion(version: MetamodelVersion) { @@ -176,6 +209,42 @@ class DrivineMetamodelVersionStore( mapOf("schemaName" to schemaName, "contentHash" to contentHash), ).firstOrNull() + /** + * Overridden so the reconciled baseline is tracked independently of write order; see this class's + * own doc. Also saves [version] into the ordinary history, the way the interface default does, so + * a caller that only ever calls this for a brand-new stamp still gets it stored — both writes run + * in the one transaction, so a reader never observes the pointer moved without the stamp it names + * being resolvable. + */ + @Transactional + override fun markSwept(version: MetamodelVersion) { + saveVersion(version) + logger.debug( + "Marking metamodel version schemaName={} contentHash={} as the reconciled baseline", + version.schemaName, + version.contentHash.take(8), + ) + persistenceManager.execute( + QuerySpecification.withStatement(MARK_SWEPT) + .bind(mapOf("schemaName" to version.schemaName, "contentHash" to version.contentHash)), + ) + } + + /** + * Overridden to resolve the reconciled baseline from [markSwept]'s own pointer. The interface + * default answers `latestVersion`, a write-order question that gets the wrong answer here — see + * this class's own doc. + */ + @Transactional(readOnly = true) + override fun sweptVersion(schemaName: String): MetamodelVersion? { + val sweptContentHash = persistenceManager.maybeGetOne( + QuerySpecification.withStatement(SWEPT_CONTENT_HASH) + .bind(mapOf("schemaName" to schemaName)) + .transform(String::class.java), + ) ?: return null + return findVersion(schemaName, sweptContentHash) + } + /** * Run one of the version queries and turn its rows into stamps, dropping any row that won't * deserialize. diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt index f41dd9fe..25379c85 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt @@ -20,6 +20,7 @@ import com.embabel.dice.metamodel.ObservedSchema import com.embabel.dice.metamodel.ObservedSchemaSource import org.drivine.manager.PersistenceManager import org.drivine.query.QuerySpecification +import org.springframework.transaction.annotation.Transactional import java.time.Clock /** @@ -200,9 +201,49 @@ open class DrivineObservedSchemaSource( """.trimIndent() } + /** + * Both branches issue several queries that get assembled into one [ObservedSchema], and this + * annotation is what keeps them from running as separate implicit transactions: everything below + * runs inside one Neo4j transaction, the same pattern [DrivineCollectorTraceStore.findEdgesByRun] + * uses to rehydrate one answer out of more than one query. Where these queries run as separate + * implicit transactions, a concurrent graph write landing between two of them shows up in only + * one, the other having already run by the time it committed, and the resulting [ObservedSchema] + * describes a combination of graph states that existed at no single instant. + * + * The guarantee this buys stops short of full snapshot isolation. Neo4j's default isolation + * level is read committed, a per-row guarantee that applies within a single statement's own + * execution: a single statement's own result rows are not guaranteed to reflect one coherent instant + * of the graph either, because a write can commit while that one statement is still streaming + * its rows, so a query can itself see a non-repeatable, missing, or double read of data it + * touches more than once during its own execution, on top of the cross-statement residual below. + * A write that commits while this transaction is still open can reach any statement, or any row + * within a statement, that is read after that commit, while an earlier read in the same + * transaction has already returned its own answer from the graph as it stood beforehand. What + * this annotation buys is narrowing the exposure down to the span of one transaction, and ruling + * out reads that were never even in the same transaction to begin with; it does not make any one + * statement's own result set internally coherent. A caller needing a stronger guarantee would need + * Neo4j's explicit locking, which this observation has no reason to pay for: drift is inherently a + * live-graph snapshot judged moments after the fact, and an occasional narrow race landing inside + * one check's transaction is a smaller, self-healing problem — the next check reads it either way. + * Reads spread across separate transactions carry no time bound between them at all, which is the + * wider exposure this closes. + */ + @Transactional(readOnly = true) override fun observe(contextId: ContextId?): ObservedSchema = if (contextId == null) observeWholeGraph() else observeContext(contextId) + /** + * Carries its own [Transactional] annotation, which is why this override exists at all. [ObservedSchemaSource.observe]'s default body, `= observe(null)`, calls + * `observe(contextId)` on `this` from inside the bean's own compiled code — a self-invocation. + * Spring's proxy applies `@Transactional` advice only to calls that arrive through the proxy + * from outside the bean, so a caller invoking the interface's no-argument `observe()` on this + * class, absent this override, would reach [observeWholeGraph] with no transaction started at + * all, and its several queries would run as separate implicit transactions again, exactly + * what annotating the two-argument overload was meant to close. + */ + @Transactional(readOnly = true) + override fun observe(): ObservedSchema = observe(null) + private fun observeWholeGraph(): ObservedSchema { // Subtract only the bookkeeping the domain has not also claimed. A name both dice and the // domain use counts as the domain's here, so it stays observable as drift. @@ -213,6 +254,7 @@ open class DrivineObservedSchemaSource( entityTypeNames = queryStrings(ALL_LABELS) - hiddenLabels, relationshipTypeNames = queryStrings(ALL_RELATIONSHIP_TYPES) - hiddenRelationshipTypes, capturedAt = clock.instant(), + entityTypeBasis = ObservedSchema.EntityTypeBasis.GRAPH_LABELS, ) } @@ -223,10 +265,16 @@ open class DrivineObservedSchemaSource( // `sourcePropositions`, which only a projected domain edge carries. Subtracting names on top // would drop a domain relationship type spelled `DERIVED_FROM` that the query had already // shown to be the domain's. + // + // Tagged MENTION_TYPES deliberately: a mention's `type` is domain data an extractor wrote, + // living in its own namespace apart from graph labels, so a governed type's inherited + // parent label must stay out of what + // counts as declared for it. See ObservedSchema.EntityTypeBasis. return ObservedSchema( entityTypeNames = queryStrings(MENTION_TYPES_IN_CONTEXT, bindings), relationshipTypeNames = queryStrings(RELATIONSHIP_TYPES_IN_CONTEXT, bindings), capturedAt = clock.instant(), + entityTypeBasis = ObservedSchema.EntityTypeBasis.MENTION_TYPES, ) } diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt index 4d3ed31d..e62b0b48 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt @@ -46,6 +46,7 @@ import org.drivine.query.QuerySpecification import org.drivine.query.dsl.* import org.slf4j.LoggerFactory import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.TransactionDefinition import org.springframework.transaction.annotation.Propagation import org.springframework.transaction.annotation.Transactional import org.springframework.transaction.support.TransactionSynchronizationManager @@ -124,8 +125,26 @@ class DrivinePropositionRepository( private val logger = LoggerFactory.getLogger(DrivinePropositionRepository::class.java) - /** Runs the dedup find-then-insert as one programmatic transaction (see [save]). */ - private val txTemplate = TransactionTemplate(transactionManager) + /** + * Runs the dedup find-then-insert as one programmatic transaction (see [save]). + * + * Set to [TransactionDefinition.PROPAGATION_REQUIRES_NEW], overriding `TransactionTemplate`'s own + * default of `PROPAGATION_REQUIRED`: this class also carries a class-level [Transactional], so an + * external call to [save] already has an ambient transaction by the time this template runs. A + * `REQUIRED` template would join that ambient transaction, and the actual commit would then + * happen only when the *outer* transaction commits — after `save`'s `synchronized` block has + * already released the stripe lock — which is exactly the window [save]'s own KDoc says cannot + * exist. `REQUIRES_NEW` suspends whatever ambient transaction is open and commits this one + * independently before `execute` returns, restoring that guarantee regardless of how [save] is + * called. It also keeps the constraint-violation recovery block usable: a `RuntimeException` + * thrown while participating in the ambient transaction marks it rollback-only, and a further + * `REQUIRED` use of this template would fail immediately with `UnexpectedRollbackException`, + * skipping the recovery read entirely; `REQUIRES_NEW` starts a fresh transaction unaffected by + * the doomed ambient one. + */ + private val txTemplate = TransactionTemplate(transactionManager).apply { + propagationBehavior = TransactionDefinition.PROPAGATION_REQUIRES_NEW + } /** * Striped locks for save-time exact-text dedup. Bounded (no per-key leak): a proposition's diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt index f865dcc7..449832cd 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt @@ -137,4 +137,88 @@ abstract class AbstractMetamodelVersionStoreContractTest { fun `latestVersion is null for a schema with no versions`() { assertNull(store().latestVersion("contract-never-saved")) } + + // ---- the swept baseline, tracked apart from ordinary write order ---- + // + // A store that doesn't override `sweptVersion`/`markSwept` inherits the interface default, + // forwarding to `latestVersion`/`saveVersion` — see `MetamodelVersionStore.sweptVersion`'s doc + // for why that reopens the exact bug `DefaultDriftCheckRunner` relies on this pointer to close. + // Three of the four tests below fail against that default: the null-until-swept case, the + // independence-from-a-later-new-stamp case, and the independence-from-a-later-re-save case. The + // middle test, `markSwept moves the swept baseline to that version`, passes against the + // forwarding default too — a schema with exactly one saved version has that version as both its + // `latestVersion` and (via `markSwept`'s forwarding to `saveVersion`) its only candidate, so the + // default answers correctly by coincidence on a single-version schema. It stays in the suite as a + // positive check on the real behavior; it just isn't the one that catches a non-overriding store. + + @Test + fun `sweptVersion is null until markSwept is called`() { + val store = store() + val schemaName = "contract-swept-null" + + store.saveVersion(version(schemaName)) + + assertNull( + store.sweptVersion(schemaName), + "an ordinary save must not look like a completed sweep", + ) + } + + @Test + fun `markSwept moves the swept baseline to that version`() { + val store = store() + val schemaName = "contract-swept-moves" + val stamp = version(schemaName) + store.saveVersion(stamp) + + store.markSwept(stamp) + + assertEquals(stamp, store.sweptVersion(schemaName)) + } + + @Test + fun `saveVersion alone never moves an already-established swept baseline`() { + // The exact case a forwarding default gets wrong: every run re-stamps its declaration, dry, + // scoped, or crashed alike, and none of those is a completed reconciliation. + val store = store() + val schemaName = "contract-swept-independent" + val reconciled = version(schemaName, "Reconciled") + store.saveVersion(reconciled) + store.markSwept(reconciled) + + store.saveVersion(version(schemaName, "NotYetReconciled")) + + assertEquals( + reconciled, + store.sweptVersion(schemaName), + "an ordinary save must not advance the reconciled baseline on its own", + ) + } + + @Test + fun `re-saving an existing, unreconciled stamp never moves the swept baseline either`() { + // The case above saves `NotYetReconciled` exactly once, as a brand-new stamp; the only + // re-save it performs is of `reconciled` itself, through `markSwept`. Here `notYetReconciled` + // gets saved a second time — an ordinary re-save of a stamp that already exists but was never + // swept — which is the gap that case leaves open. A store that keys "advance the swept + // pointer" off any save landing on an existing key, mistaking an ordinary re-save for a sign + // that stamp is now reconciled, would pass that case and still be wrong: PR #86's design note + // calls out this exact re-save hazard for saveVersion's own history contract, and the swept + // pointer needs the same guard. + val store = store() + val schemaName = "contract-swept-resave" + val reconciled = version(schemaName, "Reconciled") + val notYetReconciled = version(schemaName, "NotYetReconciled") + store.saveVersion(reconciled) + store.markSwept(reconciled) + store.saveVersion(notYetReconciled) + + store.saveVersion(notYetReconciled) + + assertEquals( + reconciled, + store.sweptVersion(schemaName), + "re-saving an existing, unreconciled stamp must not advance the reconciled baseline", + ) + } } diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt index 10c87aa9..d792ec12 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt @@ -87,6 +87,10 @@ class DrivineDriftCheckIntegrationTest { relationshipNames = emptyList(), ) + // StructuralMetamodelDiffer implements both differ interfaces; one instance plays both roles, + // the way DefaultDriftCheckRunner's own doc says it ordinarily does. + private val differ = StructuralMetamodelDiffer() + private val runner: DriftCheckRunner by lazy { DefaultDriftCheckRunner( declaredSchemaSource = DeclaredSchemaSource { @@ -94,7 +98,8 @@ class DrivineDriftCheckIntegrationTest { }, versionStore = versionStore, observedSchemaSource = observedSchemaSource, - differ = StructuralMetamodelDiffer(), + differ = differ, + metamodelDiffer = differ, driftReportStore = reportStore, quarantinePolicy = MentionTypeDriftQuarantinePolicy(), propositionStore = repository, diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt index ec270525..ba7aefca 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt @@ -18,6 +18,7 @@ package com.embabel.dice.storage import com.embabel.agent.core.ContextId import com.embabel.dice.metamodel.DriftReport import com.embabel.dice.metamodel.MetamodelVersion +import com.embabel.dice.metamodel.ObservedSchema import org.drivine.manager.PersistenceManager import org.drivine.query.QuerySpecification import org.junit.jupiter.api.AfterEach @@ -110,6 +111,17 @@ class DrivineObservedSchemaSourceIntegrationTest { assertEquals(setOf("Works_At_Company"), source.observe(tenantA).relationshipTypeNames) } + @Test + fun `a scoped observation is tagged with the mention-types basis`() { + // A mention's type is domain data an extractor wrote, living in its own namespace apart + // from graph labels; the differ needs + // this tag to know it must not widen the declared side to a governed type's inherited labels. + writeProposition("p-a", tenantA) + writeMention("p-a", "m-a", type = "Person") + + assertEquals(ObservedSchema.EntityTypeBasis.MENTION_TYPES, source.observe(tenantA).entityTypeBasis) + } + @Test fun `an empty context observes nothing rather than the whole graph`() { writeProposition("p-a", tenantA) @@ -132,6 +144,7 @@ class DrivineObservedSchemaSourceIntegrationTest { assertTrue(observed.entityTypeNames.contains("Entity"), "got ${observed.entityTypeNames}") assertTrue(observed.relationshipTypeNames.contains("GLOBAL_REL"), "got ${observed.relationshipTypeNames}") + assertEquals(ObservedSchema.EntityTypeBasis.GRAPH_LABELS, observed.entityTypeBasis) } @Test diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceTransactionIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceTransactionIntegrationTest.kt new file mode 100644 index 00000000..9f041867 --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceTransactionIntegrationTest.kt @@ -0,0 +1,114 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import org.drivine.manager.PersistenceManager +import org.drivine.query.QuerySpecification +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Primary +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.springframework.transaction.support.TransactionSynchronizationManager +import java.util.concurrent.CopyOnWriteArrayList + +/** + * Delegates every call to [delegate] unchanged, and records whether a Spring-managed transaction + * was active at the moment each [query] call ran. This is the probe the test class below uses to + * tell whether [DrivineObservedSchemaSource]'s several queries genuinely share one transaction or + * each opens its own: [TransactionSynchronizationManager.isActualTransactionActive] is `true` only + * while a `@Transactional` method's advice has a transaction open on the calling thread. + */ +class TransactionRecordingPersistenceManager(private val delegate: PersistenceManager) : PersistenceManager by delegate { + + val transactionActiveDuringQuery: MutableList = CopyOnWriteArrayList() + + override fun query(spec: QuerySpecification): List { + transactionActiveDuringQuery += TransactionSynchronizationManager.isActualTransactionActive() + return delegate.query(spec) + } +} + +/** + * Marks [TransactionRecordingPersistenceManager] as the [Primary] `PersistenceManager` in this + * test class's own Spring context, wrapping the real one [TestApplication] supplies. Every bean + * that asks for a plain [PersistenceManager] — including [DrivineObservedSchemaSource] itself — + * receives the recording wrapper instead, so its query calls are the ones under test. + * + * A distinct `@SpringBootTest(classes = [...])` combination gets its own cached Spring context, so + * this substitution cannot leak into the other Drivine integration tests, which use + * `TestApplication` alone. + */ +@TestConfiguration +open class RecordingPersistenceManagerConfig { + + @Bean + @Primary + open fun recordingPersistenceManager( + @Qualifier("persistenceManager") real: PersistenceManager, + ): TransactionRecordingPersistenceManager = TransactionRecordingPersistenceManager(real) +} + +/** + * Proves that [DrivineObservedSchemaSource]'s no-argument, whole-graph `observe()` entry point + * runs its queries inside one transaction, the same way the context-scoped overload already does. + * + * The no-argument overload is [com.embabel.dice.metamodel.ObservedSchemaSource.observe]'s default + * body, `= observe(null)`, a self-invocation on `this`. Annotating only `observe(contextId)` with + * `@Transactional` leaves that self-invocation unadvised — Spring's proxy never sees it — so a + * caller reaching this class through the no-argument overload alone would still run every one of + * [DrivineObservedSchemaSource]'s whole-graph queries as a separate implicit transaction, exactly + * the bug annotating the two-argument overload was meant to close. This is why + * [DrivineObservedSchemaSource] carries its own `@Transactional`-annotated override of the + * no-argument `observe()`. + */ +@SpringBootTest(classes = [TestApplication::class, RecordingPersistenceManagerConfig::class]) +class DrivineObservedSchemaSourceTransactionIntegrationTest { + + companion object { + @JvmStatic + @DynamicPropertySource + fun neo4jProperties(registry: DynamicPropertyRegistry) = Neo4jTestContainer.registerProperties(registry) + } + + @Autowired + private lateinit var source: DrivineObservedSchemaSource + + @Autowired + private lateinit var recordingPersistenceManager: TransactionRecordingPersistenceManager + + @Test + fun `the no-argument observe() runs its whole-graph queries inside one active transaction`() { + // Schema setup and other beans' own startup queries run before this point; only what + // observe() itself issues is relevant. + recordingPersistenceManager.transactionActiveDuringQuery.clear() + + source.observe() + + val recorded = recordingPersistenceManager.transactionActiveDuringQuery + assertTrue(recorded.isNotEmpty(), "expected observe() to issue at least one query") + assertTrue( + recorded.all { it }, + "every query the no-argument observe() issues must run inside an active transaction, " + + "but saw $recorded", + ) + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionRepositoryDedupRaceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionRepositoryDedupRaceIntegrationTest.kt new file mode 100644 index 00000000..8cfdd7ae --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionRepositoryDedupRaceIntegrationTest.kt @@ -0,0 +1,324 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import com.embabel.agent.core.ContextId +import com.embabel.dice.proposition.EntityMention +import com.embabel.dice.proposition.Proposition +import org.drivine.manager.PersistenceManager +import org.drivine.query.QuerySpecification +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Assertions.fail +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Primary +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource +import org.springframework.transaction.support.TransactionSynchronization +import org.springframework.transaction.support.TransactionSynchronizationManager +import java.lang.management.ManagementFactory +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference + +/** + * Delegates every call to [delegate] unchanged, except that the first [maybeGetOne] call after + * [arm] records the calling thread as the writer and registers a [TransactionSynchronization] on + * whichever physical transaction is current — the dedup transaction's own, whether that is + * `save`'s [DrivinePropositionRepository.txTemplate] committing independently or the class-level + * ambient one it has joined. That callback fires right before the real commit, pausing there until + * [proceed] is signaled, at exactly the moment the test needs to pin. + * + * The first later [maybeGetOne] call from a *different* thread — the sibling's own existence check + * — is let through to [delegate] first, and only *after it returns* does this record + * [siblingReadResult] and count down [siblingReadCompleted]. That ordering is the whole point: the + * signal must mean "the read finished", not "the read started", or a test racing to react to it + * could release the writer before the sibling's statement has actually reached the database — the + * exact gap that let an earlier version of this class's test pass against the regression by luck. + * [siblingReadResult] then says what the sibling actually saw: null is the pre-commit read this + * test exists to force; non-null this early would mean the writer's pause did not hold. + * + * Single-shot by design (guarded by [armed]): only the first thread through the dedup path arms + * the hook. The writer's own second `maybeGetOne` call (`existsById`, right after `findDuplicateId` + * in [DrivinePropositionRepository.findOrPersist]) runs on the same thread that armed it, so it + * does not get mistaken for the sibling's. + */ +class RaceWindowPersistenceManager(private val delegate: PersistenceManager) : PersistenceManager by delegate { + + private val armed = AtomicBoolean(false) + private val siblingReadCaptured = AtomicBoolean(false) + private val writerThread = AtomicReference(null) + private var writerPaused: CountDownLatch? = null + private var proceed: CountDownLatch? = null + private var siblingReadCompleted: CountDownLatch? = null + + /** What the sibling's first read actually returned, set once [siblingReadCompleted] fires. */ + @Volatile + var siblingReadResult: Any? = null + private set + + /** + * [writerPaused] fires the instant the writer is genuinely paused pre-commit, still inside + * `save`'s `synchronized` block — the test must wait for it before it submits the sibling task + * at all, or which of the two callers actually reaches the stripe lock first is a race in its + * own right, undoing the rest of this class's determinism. + */ + fun arm(writerPaused: CountDownLatch, proceed: CountDownLatch, siblingReadCompleted: CountDownLatch) { + armed.set(false) + siblingReadCaptured.set(false) + writerThread.set(null) + siblingReadResult = null + this.writerPaused = writerPaused + this.proceed = proceed + this.siblingReadCompleted = siblingReadCompleted + } + + override fun maybeGetOne(spec: QuerySpecification): T? { + if (armed.compareAndSet(false, true)) { + writerThread.set(Thread.currentThread()) + TransactionSynchronizationManager.registerSynchronization(object : TransactionSynchronization { + override fun beforeCommit(readOnly: Boolean) { + writerPaused?.countDown() + // The 30s bound here is a last-resort anti-hang guard only, well past the ~10s + // window the test polls on its own — under normal execution the test's explicit + // proceed.countDown() always fires first, driven by a directly observed fact. + // See the test class KDoc for why that distinction matters. + proceed?.await(30, TimeUnit.SECONDS) + } + }) + return delegate.maybeGetOne(spec) + } + val isSibling = Thread.currentThread() !== writerThread.get() + val result = delegate.maybeGetOne(spec) + if (isSibling && siblingReadCaptured.compareAndSet(false, true)) { + siblingReadResult = result + siblingReadCompleted?.countDown() + } + return result + } +} + +/** + * Marks [RaceWindowPersistenceManager] as the [Primary] `PersistenceManager` in this test class's + * own Spring context, wrapping the real one [TestApplication] supplies — the same substitution + * technique [DrivineObservedSchemaSourceTransactionIntegrationTest] uses, in a distinct + * `@SpringBootTest` combination so it cannot leak into the other Drivine integration tests. + */ +@TestConfiguration +open class RaceWindowPersistenceManagerConfig { + + @Bean + @Primary + open fun raceWindowPersistenceManager( + @Qualifier("persistenceManager") real: PersistenceManager, + ): RaceWindowPersistenceManager = RaceWindowPersistenceManager(real) +} + +/** + * The concurrent counterpart to `DrivinePropositionStoreIntegrationTest`'s sequential dedup test + * and its multi-waiter throughput test. `save`'s own KDoc says the stripe lock is held across the + * transaction commit, so a same-stripe sibling can never read pre-commit and slip a duplicate past + * the existence check. Neither of those other two tests can pin that specific claim: a free-running + * pool of callers proves dedup holds under load, but nothing forces any one of them to actually + * attempt a read during the narrow window between lock release and commit, so a regression there + * can pass by favorable scheduling as easily as it can be caught. + * + * This test removes the luck with an explicit handshake built on positive facts. [RaceWindowPersistenceManager] + * pins the writer inside [TransactionSynchronization.beforeCommit] — after its own dedup + * transaction's real commit is ready to run but before it actually runs. Once the sibling is + * submitted, the test polls for one of two *directly observed* facts, always something that + * actually happened: either the sibling's own `maybeGetOne` call *completes* (proof its read ran to + * completion, statement and all, while the writer's commit was still paused — the regression), or a + * JMX thread dump shows the sibling genuinely blocked entering the identical stripe-lock monitor + * `save` itself would use for this `(contextId, text)` pair, and that specific lock alone (proof + * the fix's mutual exclusion is what's holding it back). Only once one of those two facts is + * confirmed does the test release the writer — reacting to the read merely *starting* would still + * leave the actual database round trip racing the writer's release, so the signal fires only once + * [RaceWindowPersistenceManager.siblingReadResult] is already captured. + * + * The thread-dump check is deliberately narrower than a bare `Thread.State.BLOCKED` read: a thread + * can report `BLOCKED` for reasons that have nothing to do with this test (a connection-pool + * checkout, for instance), and treating any such incidental block as proof of the *specific* + * stripe-lock contention this test is pinning would reopen exactly the ambiguity this rewrite + * exists to close. Comparing the JMX lock owner's identity against the actual monitor + * [DrivinePropositionRepository.lockFor] would hand the sibling (read via reflection, since the + * field is private and this is the only way to name that specific object from a test) is what + * turns "blocked on our lock, specifically" into a verified claim. A bare `Future` timeout — the + * previous version of this test — cannot tell "the sibling hasn't been scheduled yet" apart from + * "the sibling is intentionally blocked": both look like nothing happening yet, which is exactly + * what let the regression's mutation test pass by favorable scheduling in that earlier version. + * + * That handshake deterministically distinguishes both outcomes: with `txTemplate`'s commit boundary + * independent of the ambient transaction (the shipped fix), the sibling blocks entering the stripe + * lock's monitor and cannot even begin its read until the writer is released — the test confirms + * that specific lock ownership, releases the writer, and the sibling then reads the now-committed + * proposition. With the boundary joined to the ambient transaction (the regression), the lock is + * already free by the time the hook fires, so the sibling's read runs to completion immediately, + * finds nothing, and is observed directly with its empty result captured — the test releases the + * writer only after that read has already finished, so the sibling genuinely saw nothing committed + * before it went on to write its own duplicate. This is one deterministic run with no survival + * count to justify. + * + * What each arm of the wait proves, stated plainly: a captured [RaceWindowPersistenceManager.siblingReadResult] + * of `null` proves the sibling's existence check ran to completion, over the network, and found + * nothing while the writer's commit was still pending — the regression, reproduced. A JMX-confirmed + * block on the exact stripe-lock monitor proves the sibling cannot even begin that same check yet — + * the fix, holding. Neither is inferred from an elapsed timeout with nothing else to show for it; + * the test fails outright, loudly, if it observes neither within its wait window, or if it observes + * a completed sibling read that is *not* empty (a sign the writer's pause did not hold). + */ +@SpringBootTest(classes = [TestApplication::class, RaceWindowPersistenceManagerConfig::class]) +class DrivinePropositionRepositoryDedupRaceIntegrationTest { + + companion object { + @JvmStatic + @DynamicPropertySource + fun neo4jProperties(registry: DynamicPropertyRegistry) = Neo4jTestContainer.registerProperties(registry) + + /** + * The private lock [DrivinePropositionRepository.save] itself would take for this pair, read via + * reflection on the real target — not the CGLIB proxy `repository` autowires. The proxy shell + * never runs its own field initializers (Spring builds it without calling the target's + * constructor), so [DrivinePropositionRepository.dedupLocks] is null there; only the actual + * target instance the proxy delegates to has the array `save` itself indexes into. + */ + private fun stripeLockFor(repository: DrivinePropositionRepository, contextId: String, text: String): Any { + val target = (repository as org.springframework.aop.framework.Advised) + .targetSource.target as DrivinePropositionRepository + val method = DrivinePropositionRepository::class.java.getDeclaredMethod( + "lockFor", String::class.java, String::class.java, + ) + method.isAccessible = true + return method.invoke(target, contextId, text) + } + + /** True only if [thread] is JMX-reported as `BLOCKED` entering [monitor] specifically — not any lock. */ + private fun isBlockedOn(thread: Thread, monitor: Any): Boolean { + val info = ManagementFactory.getThreadMXBean().getThreadInfo(thread.threadId(), 1) ?: return false + if (info.threadState != Thread.State.BLOCKED) return false + val lockInfo = info.lockInfo ?: return false + return lockInfo.identityHashCode == System.identityHashCode(monitor) && + lockInfo.className == monitor.javaClass.name + } + } + + @Autowired + private lateinit var repository: DrivinePropositionRepository + + @Autowired + private lateinit var racePersistenceManager: RaceWindowPersistenceManager + + @Autowired + private lateinit var persistenceManager: PersistenceManager + + @AfterEach + fun cleanUp() { + repository.clearAll() + persistenceManager.execute(QuerySpecification.withStatement("MATCH (s:Source) DETACH DELETE s")) + } + + private fun prop(text: String): Proposition = Proposition( + contextId = ContextId("ctx"), + text = text, + mentions = emptyList(), + confidence = 0.9, + ) + + @Test + fun `a sibling forced to read while the writer sits between lock release and commit still dedups to one proposition`() { + val text = "Rod visited Sydney" + val contextId = "ctx" + val stripeLock = stripeLockFor(repository, contextId, text) + + val writerPaused = CountDownLatch(1) + val proceed = CountDownLatch(1) + val siblingReadCompleted = CountDownLatch(1) + racePersistenceManager.arm(writerPaused, proceed, siblingReadCompleted) + + val pool = Executors.newFixedThreadPool(2) + val siblingThread = AtomicReference(null) + try { + val writer = pool.submit { repository.save(prop(text)) } + + // Must confirm the writer is genuinely paused, holding the stripe lock, before the sibling + // is even submitted — otherwise which of the two reaches the lock first is its own + // unforced race, and the rest of this handshake would be pinned to the wrong thread. + assertTrue( + writerPaused.await(10, TimeUnit.SECONDS), + "writer never reached its pre-commit pause — the dedup path no longer goes through maybeGetOne", + ) + + val sibling = pool.submit { + siblingThread.set(Thread.currentThread()) + repository.save(prop(text)) + } + + // Poll for one of two directly observed facts — never inferred from elapsed time alone. + // The read-completed arm only counts if the captured result is empty: a completed read + // that already found a row would mean the writer's pause did not actually hold, which is + // neither this test's regression case nor its fix case, and must not be read as either. + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10) + var siblingPreCommitReadObserved = false + var siblingBlockedOnStripeLock = false + while (System.nanoTime() < deadline) { + if (siblingReadCompleted.await(0, TimeUnit.MILLISECONDS)) { + val seenByReader = racePersistenceManager.siblingReadResult + if (seenByReader != null) { + fail( + "the sibling's read completed and already found a row ($seenByReader) while " + + "the writer was supposed to still be paused — the writer's pause did not hold", + ) + } + siblingPreCommitReadObserved = true + break + } + val t = siblingThread.get() + if (t != null && isBlockedOn(t, stripeLock)) { + siblingBlockedOnStripeLock = true + break + } + Thread.sleep(2) + } + if (!siblingPreCommitReadObserved && !siblingBlockedOnStripeLock) { + fail( + "the sibling neither completed a pre-commit read nor was confirmed blocked on the " + + "stripe lock within 10s — cannot tell whether the race window was actually exercised", + ) + } + + proceed.countDown() + + val writerResult = writer.get(15, TimeUnit.SECONDS) + val siblingResult = sibling.get(15, TimeUnit.SECONDS) + + assertEquals(1, repository.count(), "the forced-overlap sibling must still dedup to one proposition") + assertTrue( + writerResult.text == text && siblingResult.text == text, + "both saves must resolve to the deduped proposition's text", + ) + } finally { + pool.shutdownNow() + } + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt index 6472c26d..e7e14317 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivinePropositionStoreIntegrationTest.kt @@ -56,6 +56,7 @@ import org.springframework.transaction.annotation.AnnotationTransactionAttribute import org.springframework.transaction.interceptor.TransactionInterceptor import java.time.Duration import java.time.Instant +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit @@ -196,6 +197,57 @@ class DrivinePropositionStoreIntegrationTest { assertEquals(1, repository.count()) } + /** + * The multi-waiter counterpart to the sequential dedup test above: many same-stripe callers, a + * different crowd size than `DrivinePropositionRepositoryDedupRaceIntegrationTest`'s + * deterministic race test pins. That other test is what proves the propagation regression + * window is closed — a two-party handshake, by design, since forcing a specific window needs + * an exact pair to pin. It says nothing on its own about a bigger crowd: whether the stripe + * lock genuinely serialises every waiter, beyond the one pair it controls, and whether + * throughput under contention still converges on one proposition. That is this test's job, run + * under ordinary scheduling, with no forced window. + */ + @Test + fun `many concurrent saves of identical text still dedup to exactly one proposition`() { + val callers = 16 + val startTogether = CountDownLatch(1) + val savedIds = CopyOnWriteArrayList() + val failures = CopyOnWriteArrayList() + val pool = Executors.newFixedThreadPool(callers) + try { + repeat(callers) { + pool.submit { + startTogether.await() + runCatching { repository.save(prop("Rod visited Sydney")) } + .onSuccess { savedIds += it.id } + .onFailure { failures += it } + } + } + startTogether.countDown() + pool.shutdown() + assertTrue(pool.awaitTermination(60, TimeUnit.SECONDS), "concurrent saves did not finish in time") + } finally { + pool.shutdownNow() + } + + assertTrue( + failures.isEmpty(), + "every save must resolve to a proposition without throwing, but $callers callers produced " + + "${failures.size} failure(s): ${failures.firstOrNull()}", + ) + assertEquals( + 1, + repository.count(), + "$callers concurrent saves of identical text must leave exactly one proposition; found " + + "${repository.count()} (ids returned: ${savedIds.toSet()})", + ) + } + + // The deterministic counterpart to both dedup tests above — the one that actually pins the + // propagation-regression window under a forced, controlled overlap — lives in + // DrivinePropositionRepositoryDedupRaceIntegrationTest.kt, which needs its own decorated + // PersistenceManager bean (a different Spring context) to force that window on demand. + @Test fun `query pushes filters incl entity quantifier`() { repository.save(prop("a", entityId = "e1", status = PropositionStatus.ACTIVE)) diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt index 15fc0062..0aed179b 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt @@ -89,6 +89,7 @@ class PinnableClock : Clock() { @EnableDrivine @EnableDrivineTestConfig @EnableAspectJAutoProxy(proxyTargetClass = true) +@org.springframework.transaction.annotation.EnableTransactionManagement open class TestApplication { @Bean diff --git a/docs/design/metamodel-drift.md b/docs/design/metamodel-drift.md index 8857cf6f..d0e60abd 100644 --- a/docs/design/metamodel-drift.md +++ b/docs/design/metamodel-drift.md @@ -648,6 +648,30 @@ Two limits follow. Exclusion is decided per label, not per node, so a graph mixi with dice's own reports `Source` every run until the type is declared. And deciding it costs a scan of dice's own labels on each unscoped observation; context-scoped checks don't pay it. +`observe` runs its whole set of queries — bookkeeping-exclusion probes included — inside one Neo4j +transaction, so the several reads that get assembled into one `ObservedSchema` come from a single +transaction's view of the graph, closing the case where a concurrent write landing between separately +transacted queries produces a combined observation the graph never actually held at any instant. +Neo4j's default isolation level is read committed, and that guarantee holds per row, leaving both a +statement and a transaction free to see the graph shift underneath them: a write that commits mid-statement can still land in that same statement's own +result set, so a single statement can itself see a non-repeatable, missing, or double read of data +it touches more than once while it runs, and its rows are not guaranteed to reflect one coherent +instant of the graph, on top of the residual race between two different statements inside the same +transaction. This narrows the exposure to the span of one transaction and rules out reads that were +never even in the same transaction, and no more than that; the method's own KDoc states the residual +plainly. + +The observation also tags what kind of name its entity types are: `ObservedSchema.EntityTypeBasis`, +`GRAPH_LABELS` for the unscoped path and `MENTION_TYPES` for the context-scoped one. The two answer +different questions. A Neo4j label carries a type's whole declared hierarchy — `Person` with parent +`Agent` puts both labels on every `Person` node — so the differ's declared side for a `GRAPH_LABELS` +observation widens to every label a declared type carries. `Mention.type` is domain data an extractor +wrote, and a governed type's inherited label has no bearing on it, so the differ's declared side for a +`MENTION_TYPES` observation stays on declared type names and their declared former names, with no +widening. Tagging the two the same way let a mention typed `Agent` conform under a schema that only +governs `Person` with parent `Agent` — an undeclared mention type escaping detection by riding a +governed type's parent label — until the observation itself carried which comparison it needs. + Hosts declare the constraints these stores need (see `MetamodelSchema`); a MERGE is race-free only under a uniqueness constraint on the key it merges on. diff --git a/docs/design/metamodel-versioning.md b/docs/design/metamodel-versioning.md index 33ec5d2a..6b7e9d77 100644 --- a/docs/design/metamodel-versioning.md +++ b/docs/design/metamodel-versioning.md @@ -342,7 +342,7 @@ contract has an executable statement of what its rules mean; durable storage is The durable implementation lives in `dice-storage`. `DrivineMetamodelVersionStore` keeps each stamp as a `(:MetamodelVersion)` node and MERGEs on `(schemaName, contentHash)`, so re-stamping an -unchanged schema updates the node already there. Three things govern how it behaves: +unchanged schema updates the node already there. Four things govern how it behaves: - **It needs three uniqueness constraints**, declared in a `SchemaCatalog` bean. A MERGE is race-free only when what it merges on is unique, so `MetamodelVersion(schemaName, contentHash)` @@ -359,6 +359,13 @@ unchanged schema updates the node already there. Three things govern how it beha - **A re-save updates content only.** Sequence, counter, and `savedAt` keep their existing values, so an old stamp stays at its original position in the history. `InMemoryMetamodelVersionStore`, the reference implementation `dice-metamodel` ships, behaves the same way. +- **The reconciled baseline tracks independently of write order.** This store supplies its own + `sweptVersion`/`markSwept`, leaving the interface's forwarding default behind: `markSwept` writes `sweptContentHash` + as a property on the schema's own `(:MetamodelSchemaCounter)` node, and `sweptVersion` resolves that + hash back through `findVersion`. An ordinary `saveVersion` never touches it, so a dry run, a scoped + run, or a crash mid-sweep — every path [metamodel-drift.md](metamodel-drift.md) walks through — + leaves the baseline exactly where it was, the same independence `InMemoryMetamodelVersionStore` + keeps in its separate `swept` map. The structural fields are stored as JSON strings, since Neo4j properties are scalars and flat arrays. Property signatures get explicit named fields with enums by name From b4c42b05b02b7ef553c754763d9d9dff3641e350 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:49:45 -0400 Subject: [PATCH 04/10] Observe mention types globally and derive the bookkeeping exclusion The unscoped observation read graph labels only, so an undeclared mention type with no same-named projected label never reached the default drift check, and a mention typed as a governed type's inherited label passed by riding it. ObservedSchema now carries mention types as their own set: the differ judges labels under the observation's basis and mention types under the mention rule, unioning the drift, and the Drivine source fills both from one snapshot. The bookkeeping exclusion derives from the storage schema definitions, so a host domain node shaped like DICE bookkeeping stays observed. DrivineMetamodelVersionStore declares SweptBaselineStore, binding its existing swept tracking to the contract that moved there. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- CHANGELOG.md | 55 ++++- .../embabel/dice/metamodel/ObservedSchema.kt | 25 ++- .../support/StructuralMetamodelDiffer.kt | 28 ++- .../dice/metamodel/MetamodelDifferTest.kt | 60 ++++++ .../embabel/dice/storage/DiceOwnedSchema.kt | 169 ++++++++++++++++ .../storage/DrivineCollectorRecordStore.kt | 11 +- .../storage/DrivineMetamodelVersionStore.kt | 29 +-- .../storage/DrivineObservedSchemaSource.kt | 137 +++++++------ .../storage/DrivineProjectionRecordStore.kt | 8 +- .../com/embabel/dice/storage/LineageSchema.kt | 81 ++++++++ ...stractMetamodelVersionStoreContractTest.kt | 36 ++-- .../dice/storage/DiceOwnedSchemaTest.kt | 98 +++++++++ .../DrivineDriftCheckIntegrationTest.kt | 188 ++++++++++++++++-- ...odelVersionStoreContractIntegrationTest.kt | 4 +- ...vineObservedSchemaSourceIntegrationTest.kt | 182 +++++++++++++++-- ...MemoryMetamodelVersionStoreContractTest.kt | 4 +- .../embabel/dice/storage/LineageSchemaTest.kt | 56 ++++++ .../embabel/dice/storage/TestApplication.kt | 10 +- docs/design/metamodel-drift.md | 78 ++++++-- 19 files changed, 1069 insertions(+), 190 deletions(-) create mode 100644 dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt create mode 100644 dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/LineageSchemaTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 314a7aca..6e3ff53b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -389,14 +389,14 @@ and the consumer PRs that deliver it). type of its own — an inherited-label escape hatch for undeclared mention types. `DrivineObservedSchemaSource` tags its context-scoped observation `MENTION_TYPES`; `StructuralMetamodelDiffer.diffAgainstObserved` now compares a `MENTION_TYPES` observation against declared type names and their declared former - names only, with no widening to inherited labels. Third, `DrivineMetamodelVersionStore` overrides - `sweptVersion`/`markSwept`, tracking the reconciled baseline as a `sweptContentHash` property on the - schema's own `(:MetamodelSchemaCounter)` node, moved only by `markSwept` and left untouched by an - ordinary `saveVersion`, the same independence `InMemoryMetamodelVersionStore` already had. Without - this override the durable store inherited the interface's forwarding default, and every run's own - history-stamping write — dry, scoped, or crashed alike — silently consumed the very signal - `DefaultDriftCheckRunner`'s declared-vs-previous comparison depends on, a gap the drift-runner slice - above called out and deferred to this one. + names only, with no widening to inherited labels. Third, `DrivineMetamodelVersionStore` tracks the + reconciled baseline as a `sweptContentHash` property on the schema's own + `(:MetamodelSchemaCounter)` node, moved only by `markSwept` and left untouched by an ordinary + `saveVersion`, the same independence `InMemoryMetamodelVersionStore` already had. A durable store + answering that question from write order would let every run's own history-stamping write — dry, + scoped, or crashed alike — silently consume the very signal `DefaultDriftCheckRunner`'s + declared-vs-previous comparison depends on, a gap the drift-runner slice above called out and + deferred to this one. **Compatibility: additive.** `ObservedSchema` gains a defaulted constructor parameter under `@JvmOverloads`, so the pre-existing three-argument constructor survives in the compiled class alongside the new four-argument one, confirmed by running `javap` on the compiled class after @@ -405,7 +405,44 @@ and the consumer PRs that deliver it). gain behavior on existing methods; no signature changed. `AbstractMetamodelVersionStoreContractTest` gains four `sweptVersion`/`markSwept` cases; the graph store now passes all four, and three of them — the null-until-swept case, the independence-from-a-later-new-stamp case, and the - independence-from-a-later-re-save case — previously failed against the forwarding default. + independence-from-a-later-re-save case — catch a store that answers from write order. + +- Three more corrections to the same Drivine drift slice, from a later review round. + First, a whole-graph observation now asks dice's own propositions for their distinct `Mention.type` + values, alongside the label catalogue it already read. A mention type reaches `db.labels()` only + once something projects a node for it, so an extraction that recorded `Ghost` and projected nothing + left an undeclared type invisible to every unscoped check, while the context-scoped check on the + same data reported it. The query holds both ends to dice's own shape, so a domain node wearing + `:Proposition` contributes no mention types. The two kinds of name stay in separate sets: + `ObservedSchema` gains `mentionTypeNames` (empty by default), `DrivineObservedSchemaSource` fills + it on the unscoped path, and `StructuralMetamodelDiffer.diffAgainstObserved` judges labels under + the observation's basis and mention types under the `MENTION_TYPES` rule, unioning what drifted. + Merging them would have to pick one rule for both, and the label rule reopens what `MENTION_TYPES` + exists to close: a mention typed `Agent` passing under a schema that governs `Person` with parent + label `Agent` and declares no `Agent` type. An unscoped check and a scoped one now read mention + types the same strict way, pinned by a differ test and by an integration pair that puts a + `(:Person:Agent)` node in the graph and moves only the mention type between them. + Second, the ownership catalog is derived from the storage definitions themselves, in the new + `DiceOwnedSchema`, replacing the literal inventory of labels and properties the observer used to + hold. A node fragment's shape is every constructor parameter dice's writer cannot leave out + (declared non-null, with no default), so dice's `Source` shape is `key` **and** `kind`, and a + host's own `(:Source {key: ...})` stays observed where the old key-only shape hid it. A + Cypher-backed store's shape is the union of the properties its uniqueness constraints name. The new + `LineageSchema` gives the lineage stores' labels and natural keys one definition site, and both + stores build their MERGE patterns from it, so the key a record is upserted on and the key its + constraint protects cannot drift apart. + Third, `DrivineMetamodelVersionStore` declares `SweptBaselineStore`, the sub-interface the swept + baseline moved onto, so `DefaultDriftCheckRunner` reads the durable pointer described above and a + Drivine-backed host gets the declared-vs-previous half of a report once its first sweep completes. + **Compatibility: behavioral.** `ObservedSchema` gains a fifth constructor parameter, + `mentionTypeNames`, defaulted to empty under the existing `@JvmOverloads`, so every three- and + four-argument constructor form survives and any caller that never fills it gets exactly the + comparison it got before. `DiceOwnedSchema` and `LineageSchema` are new. No signature was removed + or narrowed. A whole-graph check against a populated graph can report more than it did: mention + types nothing ever projected, and a domain node sharing a dice label while missing a property dice + always writes. Both were undetected drift before, so what appears is a real finding, and a scoped + check's answer is unchanged. Hosts declaring the lineage constraints by hand can swap in + `LineageSchema.specs()`. - `dice-storage-autoconfigure` now depends on `spring-boot-transaction`. On Spring Boot 4, a `PlatformTransactionManager` bean alone does not activate `@Transactional`: the interceptor that diff --git a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt index 24dbcbfc..08ead8f5 100644 --- a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt +++ b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/ObservedSchema.kt @@ -36,12 +36,22 @@ import java.util.Objects * contributing types that come and go, so what is in the graph can drift from what was declared, * and this module needs a way to talk about that without depending on a particular graph driver. * + * A snapshot can carry both kinds of entity name at once, in two sets. A whole-graph observation + * reads labels off the database catalogue and mention types off the stored data, and those two + * questions have different declared sides, so keeping them apart is what lets each be judged by its + * own rule. [entityTypeBasis] says which kind [entityTypeNames] holds; [mentionTypeNames] is always + * mention types and needs no tag. + * * @property entityTypeNames Entity type names observed. What counts as a name depends on * [entityTypeBasis]: a Neo4j graph label, or a mention's own `type` field. * @property relationshipTypeNames Relationship type names observed in the graph. * @property capturedAt When this snapshot was taken. * @property entityTypeBasis What kind of name [entityTypeNames] holds, which decides what the * declared side of a comparison has to be. See [EntityTypeBasis]. + * @property mentionTypeNames Types the stored data claims for its entity mentions, held apart from + * [entityTypeNames] so a comparison can judge them by the mention rule whatever + * [entityTypeBasis] says. Empty when the observation had no way to ask the data, which is the + * case for any source that only reads a label catalogue. */ @ApiStatus.Experimental class ObservedSchema @JvmOverloads constructor( @@ -49,6 +59,7 @@ class ObservedSchema @JvmOverloads constructor( relationshipTypeNames: Set, val capturedAt: Instant, val entityTypeBasis: EntityTypeBasis = EntityTypeBasis.GRAPH_LABELS, + mentionTypeNames: Set = emptySet(), ) { /** @@ -68,7 +79,9 @@ class ObservedSchema @JvmOverloads constructor( /** * The `type` a mention was extracted as, read off `Mention.type` on a context-scoped - * observation. This is domain data an extractor wrote, living in its own namespace apart from graph labels, and a graph's + * observation. An observation that carries labels in [ObservedSchema.entityTypeNames] puts + * its mention types in [ObservedSchema.mentionTypeNames], where this same rule applies to + * them. This is domain data an extractor wrote, living in its own namespace apart from graph labels, and a graph's * label hierarchy has no bearing on it: a mention typed `Agent` claimed to be an `Agent`, and * that claim stands or falls on whether `Agent` is itself a declared type, whatever labels a * governed `Person` happens to carry. The declared side of a comparison against this basis @@ -89,18 +102,22 @@ class ObservedSchema @JvmOverloads constructor( val relationshipTypeNames: Set = immutableCopy(relationshipTypeNames) + val mentionTypeNames: Set = immutableCopy(mentionTypeNames) + override fun equals(other: Any?): Boolean = other is ObservedSchema && entityTypeNames == other.entityTypeNames && relationshipTypeNames == other.relationshipTypeNames && capturedAt == other.capturedAt && - entityTypeBasis == other.entityTypeBasis + entityTypeBasis == other.entityTypeBasis && + mentionTypeNames == other.mentionTypeNames - override fun hashCode(): Int = Objects.hash(entityTypeNames, relationshipTypeNames, capturedAt, entityTypeBasis) + override fun hashCode(): Int = + Objects.hash(entityTypeNames, relationshipTypeNames, capturedAt, entityTypeBasis, mentionTypeNames) override fun toString(): String = "ObservedSchema(entityTypeNames=$entityTypeNames, relationshipTypeNames=$relationshipTypeNames, " + - "capturedAt=$capturedAt, entityTypeBasis=$entityTypeBasis)" + "capturedAt=$capturedAt, entityTypeBasis=$entityTypeBasis, mentionTypeNames=$mentionTypeNames)" private companion object { diff --git a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt index b4c88211..4f7cf386 100644 --- a/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt +++ b/dice-metamodel/src/main/kotlin/com/embabel/dice/metamodel/support/StructuralMetamodelDiffer.kt @@ -33,6 +33,10 @@ import com.embabel.dice.metamodel.PropertySignature * content hash is built from, which is what makes an empty diff and an equal hash mean the same * thing. * + * An observation can carry two kinds of entity name, and [diffAgainstObserved] judges each by its + * own rule: graph labels against the declared side its basis calls for, mention types against + * declared type names and their declared former names alone. See [ObservedSchema.mentionTypeNames]. + * * Two rules it keeps throughout. Sets are compared as sets, never as a delimiter-joined projection, * because a label or property name can contain a comma or a space, which is routine when names come * from LLM extraction, and joining would collapse two different sets into a false "unchanged". @@ -184,6 +188,18 @@ class StructuralMetamodelDiffer : MetamodelDiffer, DeclaredObservedDiffer { declared.ungovernedEntityTypeNames + ownLabelsOf(declared.ungovernedEntityTypeNames) + // Mention types arrive in their own set, and they are judged by the mention rule whatever + // basis the names above carry. A whole-graph observation reports both kinds at once — labels + // off the database catalogue, mention types off the stored propositions — and one basis tag + // can only describe one of them. Judging the pair together under GRAPH_LABELS would widen the + // declared side to inherited labels for the mention half too, which is the escape hatch + // MENTION_TYPES exists to close: a mention typed `Agent` would pass under a schema that + // governs `Person` with parent label `Agent` and declares no `Agent` type at all. + val observedMentionTypes = observed.mentionTypeNames + val mentionTypesExcludedFromDrift = declaredEitherSpelling + + declared.ungovernedEntityTypeNames + + ownLabelsOf(declared.ungovernedEntityTypeNames) + // Drift is observed and never declared: orphaned data whose declaring integration is gone, // or was never registered. The opposite direction gets its own informational bucket, since a // declared type with zero instances is an ordinary state. That direction stays on the type @@ -207,11 +223,19 @@ class StructuralMetamodelDiffer : MetamodelDiffer, DeclaredObservedDiffer { return DeclaredObservedDiff( declared = declared, observedSchema = observed, - driftedEntityTypes = canonical(observedTypes - excludedFromDrift), + driftedEntityTypes = canonical( + (observedTypes - excludedFromDrift) + (observedMentionTypes - mentionTypesExcludedFromDrift), + ), driftedRelationshipTypes = canonical(observedRels - relsExcludedFromDrift), + // Data mentioning a declared type is that type being observed, the same as a graph label + // reporting it, so both sets answer this bucket. unobservedEntityTypes = canonical( declaredTypes.filterNot { - isObserved(it, declared.version.entityTypeAliases[it].orEmpty(), observedTypes) + isObserved( + it, + declared.version.entityTypeAliases[it].orEmpty(), + observedTypes + observedMentionTypes, + ) }, ), unobservedRelationshipTypes = canonical(declaredRels - observedRels), diff --git a/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt b/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt index e8a36d33..98897719 100644 --- a/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt +++ b/dice-metamodel/src/test/kotlin/com/embabel/dice/metamodel/MetamodelDifferTest.kt @@ -518,6 +518,66 @@ class MetamodelDifferTest { entityTypeBasis = ObservedSchema.EntityTypeBasis.MENTION_TYPES, ) + /** + * A declaration governing `Person`, which carries the parent label `Agent` on every node it + * writes. Nothing declares `Agent` a type of its own. + */ + private fun personCarryingParentLabelAgent(): DeclaredSchema = DeclaredSchema( + version = MetamodelVersion( + schemaName = "test", + entityTypeNames = listOf("Person"), + entityTypeLabels = mapOf("Person" to setOf("Person", "Agent")), + entityTypeProperties = mapOf("Person" to emptySet()), + relationshipNames = emptyList(), + ), + relationshipTypeNames = emptySet(), + ) + + /** A whole-graph observation: labels in one set, mention types in the other. */ + private fun observedWholeGraph(labels: Set, mentionTypes: Set): ObservedSchema = + ObservedSchema( + entityTypeNames = labels, + relationshipTypeNames = emptySet(), + capturedAt = Instant.parse("2026-01-01T00:00:00Z"), + entityTypeBasis = ObservedSchema.EntityTypeBasis.GRAPH_LABELS, + mentionTypeNames = mentionTypes, + ) + + @Test + fun `a parent label a declared type carries is no drift`() { + val diff = declaredObservedDiffer.diffAgainstObserved( + personCarryingParentLabelAgent(), + observedWholeGraph(labels = setOf("Person", "Agent"), mentionTypes = setOf("Person")), + ) + + assertFalse(diff.hasDrift, "a governed type's own hierarchy is declared; got ${diff.driftedEntityTypes}") + } + + @Test + fun `a mention type spelled like a parent label is drift`() { + // What the two sets keep apart. The graph legally reports `Agent` as a label, since every + // governed `Person` node carries it, while a mention claiming to BE an `Agent` is a claim + // about a type nothing declared. Judging both sets under the label rule would let the + // mention ride the label through. + val diff = declaredObservedDiffer.diffAgainstObserved( + personCarryingParentLabelAgent(), + observedWholeGraph(labels = setOf("Person", "Agent"), mentionTypes = setOf("Person", "Agent")), + ) + + assertEquals(setOf("Agent"), diff.driftedEntityTypes) + } + + @Test + fun `a declared type observed only through its mentions counts as observed`() { + val diff = declaredObservedDiffer.diffAgainstObserved( + declared("Person"), + observedWholeGraph(labels = emptySet(), mentionTypes = setOf("Person")), + ) + + assertTrue(diff.unobservedEntityTypes.isEmpty(), "got ${diff.unobservedEntityTypes}") + assertFalse(diff.hasDrift, "got ${diff.driftedEntityTypes}") + } + @Test fun `an observed type with no declaration is reported as drift`() { val diff = declaredObservedDiffer.diffAgainstObserved( diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt new file mode 100644 index 00000000..298578d7 --- /dev/null +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt @@ -0,0 +1,169 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import com.embabel.dice.storage.model.Mention +import com.embabel.dice.storage.model.ProcessedChunkNode +import com.embabel.dice.storage.model.PropositionNode +import com.embabel.dice.storage.model.SourceNode +import org.drivine.annotation.NodeFragment +import org.drivine.annotation.RelationshipFragment +import org.drivine.schema.SchemaItemSpec +import org.drivine.schema.UniquenessConstraintSpec +import kotlin.reflect.KClass +import kotlin.reflect.KType +import kotlin.reflect.full.hasAnnotation +import kotlin.reflect.full.isSubclassOf +import kotlin.reflect.full.primaryConstructor + +/** + * What a node in a graph has to look like for dice to own it: the labels dice's own storage writes, + * and, per label, the properties dice writes on every such node. + * + * ## Why ownership needs saying at all + * + * A drift check compares the domain schema an app declared against what a live graph holds, and it + * shares that graph with dice's own storage. Nothing dice writes for itself belongs to a declared + * domain schema, so counting those nodes would report dice's bookkeeping as drift on every run, and + * would make governance report itself: stamping a version and writing a drift report both add nodes + * to the graph the next check observes. `DrivineObservedSchemaSource` subtracts what this object + * describes. + * + * Names alone can't decide it. An app is free to govern its own type called `Source`, and hiding + * every `(:Source)` node would hide that type from every report it should appear in. So ownership is + * decided by shape: a label counts as dice's only while every node wearing it carries the properties + * dice's own writer always writes. + * + * ## The shapes come from the schema definitions + * + * Nothing here is a hand-kept list of property names. Each shape is read out of the definition the + * store already writes from, so the two cannot disagree: + * + * - **Node fragments** ([PropositionNode], [Mention], [SourceNode], [ProcessedChunkNode]) carry + * their label in `@NodeFragment` and their properties as constructor parameters. The shape is + * every parameter dice's writer cannot leave out: declared non-null, with no default value to fall + * back on. Optional and nullable ones stay out, because a node dice wrote is allowed to be missing + * them, and demanding one would make dice's own nodes look foreign and bring back the + * self-reporting case above. + * - **Cypher-backed stores** ([MetamodelSchema], [CollectorTraceSchema], [LineageSchema]) declare + * their labels and natural keys as uniqueness constraints. The shape is the union of the key + * properties for that label, which is exactly what those stores MERGE on, so every node they + * create carries all of them. + * + * Adding a node label to any of those definitions carries its shape here with it. A new node + * fragment is the one case needing a line: add its class to [NODE_FRAGMENTS]. + * + * ## Where the boundary genuinely blurs + * + * A shape is a claim about properties, so a domain node carrying all of dice's properties for a + * label it shares is indistinguishable from dice's own. A host that keeps `(:Source {key, kind})` + * nodes of its own meaning has built dice's exact shape, and that type stays out of whole-graph + * observation until dice writes an ownership marker at persistence time, which is a data migration + * for existing graphs and a decision for a later change. Everything short of that is caught: a + * domain `(:Source {key})` with no `kind` keeps `Source` observable, as does any other domain node + * missing a property dice always writes. Context-scoped observation avoids the question entirely, + * since it reads mention types and edges marked as projected from domain data. + */ +object DiceOwnedSchema { + + /** + * The Drivine node fragments dice persists directly. Each one names its own label and carries + * its own properties, so this list holds classes and no strings. + */ + private val NODE_FRAGMENTS: List> = listOf( + PropositionNode::class, + Mention::class, + SourceNode::class, + ProcessedChunkNode::class, + ) + + /** + * Every node label dice writes, with the properties that identify a node carrying that label as + * dice's own. A node missing any of them was written by somebody else. + */ + val NODE_SHAPES: Map> = buildMap { + NODE_FRAGMENTS.forEach { fragment -> putAll(shapesOf(fragment)) } + putAll(keyShapesOf(MetamodelSchema.specs())) + putAll(keyShapesOf(CollectorTraceSchema.specs())) + putAll(keyShapesOf(LineageSchema.specs())) + } + + /** The label names, for callers that only need the names. */ + val LABELS: Set = NODE_SHAPES.keys + + /** + * A Cypher predicate that holds for a node dice owns: every property of the label's shape is + * present. + * + * Label and property names are compile-time constants of the definitions above, so nothing + * caller-derived is assembled into Cypher. + * + * @param alias The variable the node is bound to. + * @param label One of [LABELS]. + * @return The predicate text, ready to follow a `WHERE`. + */ + fun ownedNodePredicate(alias: String, label: String): String { + val shape = requireNotNull(NODE_SHAPES[label]) { "'$label' is no dice storage label" } + return shape.joinToString(" AND ") { property -> "$alias.$property IS NOT NULL" } + } + + /** The label a fragment writes, mapped to the properties dice always writes on it. */ + private fun shapesOf(fragment: KClass<*>): Map> { + val labels = fragment.annotations.filterIsInstance().singleOrNull()?.labels + ?: error("${fragment.simpleName} carries no @NodeFragment") + val shape = alwaysWritten(fragment) + return labels.associateWith { shape } + } + + /** + * The fragment's properties dice's writer cannot leave out: declared non-null, with no default + * value, and held in the node as a single property. + * + * A default value is what tells us a property is optional. Drivine's `@Default` and + * `@EmptyWhenAbsent` both go on parameters that carry one, so this covers them, along with + * `@PropertyBag`, whose map is spread across `metadata.` properties and appears under no + * name of its own. + */ + private fun alwaysWritten(fragment: KClass<*>): List { + val constructor = fragment.primaryConstructor + ?: error("${fragment.simpleName} has no primary constructor") + return constructor.parameters + .filter { parameter -> !parameter.isOptional && !parameter.type.isMarkedNullable } + .filter { parameter -> isSingleProperty(parameter.type) } + .mapNotNull { parameter -> parameter.name } + } + + /** + * Whether a value of this type lands in the node as one property. Collections, maps and nested + * fragments do something else with it, so they take no part in a shape. + */ + private fun isSingleProperty(type: KType): Boolean { + val classifier = type.classifier as? KClass<*> ?: return false + if (classifier.isSubclassOf(Collection::class) || classifier.isSubclassOf(Map::class)) return false + return !classifier.hasAnnotation() && !classifier.hasAnnotation() + } + + /** + * The labels a Cypher-backed store declares, each mapped to every property its uniqueness + * constraints name. Those properties are the store's MERGE keys, so a node it created carries + * all of them. Range indexes are left alone, since they cover properties a record can be + * missing. + */ + private fun keyShapesOf(specs: List): Map> = + specs.filterIsInstance() + .groupBy { spec -> spec.label } + .mapValues { (_, group) -> group.flatMap { spec -> spec.properties }.distinct() } +} diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorRecordStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorRecordStore.kt index dd127cae..599c5bd0 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorRecordStore.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineCollectorRecordStore.kt @@ -29,9 +29,10 @@ import org.springframework.transaction.annotation.Transactional * graph counterpart of the in-memory store, shipping here alongside [DrivinePropositionRepository]. * * The query methods default to filtering [all] / [runs] in memory, so only the writers ([record], - * [recordRun]) and readers ([all], [runs]) are supplied here. Writes MERGE on the natural key so a - * retried record updates in place rather than duplicating. Every statement is parameterized; user- - * derived values are never interpolated into Cypher. + * [recordRun]) and readers ([all], [runs]) are supplied here. Writes MERGE on the natural key + * [LineageSchema] declares, built from that same key, so a retried record updates the node already + * there and the uniqueness constraint protecting it always covers what the write matched on. Every + * statement is parameterized; user-derived values are never interpolated into Cypher. */ @Transactional class DrivineCollectorRecordStore( @@ -46,7 +47,7 @@ class DrivineCollectorRecordStore( persistenceManager.execute( QuerySpecification.withStatement( """ - MERGE (n:CollectorRecord {propositionId: ${'$'}propositionId, runId: ${'$'}runId}) + MERGE ${LineageSchema.mergePattern("n", LineageSchema.COLLECTOR_RECORD)} SET n.reason = ${'$'}reason, n.survivorId = ${'$'}survivorId, n.outcome = ${'$'}outcome, @@ -65,7 +66,7 @@ class DrivineCollectorRecordStore( persistenceManager.execute( QuerySpecification.withStatement( """ - MERGE (n:CollectorRun {runId: ${'$'}runId}) + MERGE ${LineageSchema.mergePattern("n", LineageSchema.COLLECTOR_RUN)} SET n.startedAt = ${'$'}startedAt, n.finishedAt = ${'$'}finishedAt, n.dryRun = ${'$'}dryRun diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt index 240e9851..5e2caf6e 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStore.kt @@ -17,6 +17,7 @@ package com.embabel.dice.storage import com.embabel.dice.metamodel.MetamodelVersion import com.embabel.dice.metamodel.MetamodelVersionStore +import com.embabel.dice.metamodel.SweptBaselineStore import org.drivine.manager.PersistenceManager import org.drivine.query.QuerySpecification import org.slf4j.LoggerFactory @@ -24,7 +25,8 @@ import org.springframework.transaction.annotation.Transactional import java.time.Clock /** - * Drivine/Neo4j implementation of [MetamodelVersionStore]. Every schema stamp is a + * Drivine/Neo4j implementation of [SweptBaselineStore]: a [MetamodelVersionStore] that also keeps a + * durable pointer at the declaration a sweep last finished reconciling. Every schema stamp is a * `(:MetamodelVersion)` node. * * The write MERGEs on the natural key `(schemaName, contentHash)`, so a retry or a re-stamp of an @@ -72,9 +74,11 @@ import java.time.Clock * keeps a separate `swept` map: as `sweptContentHash`, a property on the schema's own * `(:MetamodelSchemaCounter)` node, moved only by [markSwept]. Nothing about an ordinary [saveVersion] * touches it, which is what makes a dry run, a scoped run, or a crash mid-sweep leave the baseline - * exactly where it was — see `MetamodelVersionStore.sweptVersion`'s own doc for why a store that - * doesn't track this independently answers `latestVersion` instead, and gets the wrong answer once a - * declaration cycles back to a stamp it already used. + * exactly where it was. Tracking the pointer durably is what earns this store the right to declare + * [SweptBaselineStore] at all: a backend that could only answer the question from write order keeps + * the plain [MetamodelVersionStore] contract, and `DriftCheckRunner` then stays silent about a + * baseline nobody tracks. See [SweptBaselineStore.sweptVersion] for what write order gets wrong once + * a declaration cycles back to a stamp it already used. * * @param persistenceManager Drivine's handle on the `neo` datasource. * @param clock supplies the instant a version is stamped as saved at. Injectable so a test can pin @@ -84,7 +88,7 @@ import java.time.Clock class DrivineMetamodelVersionStore( private val persistenceManager: PersistenceManager, private val clock: Clock = Clock.systemUTC(), -) : MetamodelVersionStore { +) : SweptBaselineStore { private val logger = LoggerFactory.getLogger(DrivineMetamodelVersionStore::class.java) @@ -210,11 +214,10 @@ class DrivineMetamodelVersionStore( ).firstOrNull() /** - * Overridden so the reconciled baseline is tracked independently of write order; see this class's - * own doc. Also saves [version] into the ordinary history, the way the interface default does, so - * a caller that only ever calls this for a brand-new stamp still gets it stored — both writes run - * in the one transaction, so a reader never observes the pointer moved without the stamp it names - * being resolvable. + * Moves the reconciled baseline to [version], and saves the stamp into the ordinary history on + * the way, so a caller that only ever calls this for a brand-new declaration still gets it stored. + * Both writes run in the one transaction, so a reader never observes the pointer moved without the + * stamp it names being resolvable. */ @Transactional override fun markSwept(version: MetamodelVersion) { @@ -231,9 +234,9 @@ class DrivineMetamodelVersionStore( } /** - * Overridden to resolve the reconciled baseline from [markSwept]'s own pointer. The interface - * default answers `latestVersion`, a write-order question that gets the wrong answer here — see - * this class's own doc. + * Reads [markSwept]'s own pointer and resolves the hash it holds back into the stamp it names. + * A schema no sweep has ever completed for carries no such pointer, so this answers `null`; see + * this class's own doc for why write order cannot stand in for it. */ @Transactional(readOnly = true) override fun sweptVersion(schemaName: String): MetamodelVersion? { diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt index 25379c85..87fd027d 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt @@ -23,63 +23,9 @@ import org.drivine.query.QuerySpecification import org.springframework.transaction.annotation.Transactional import java.time.Clock -/** - * Every node label dice writes for its own bookkeeping, and the properties that identify a node - * carrying that label as dice's own. - * - * A drift check compares the domain schema an app declared against what a live graph holds. None of - * these labels belongs to a declared domain schema, so counting them would flag dice's own storage - * as drift on every run, and would make governance report itself: stamping a version and writing a - * report both add labels to the graph the next check observes. - * - * Exclusion is by shape, so that a domain type called `Source` stays visible. A label is excluded - * only when the nodes carrying it match dice's shape for it — dice's `Source` nodes carry `key`, its - * governance nodes carry `schemaName`, and so on. A same-named node that doesn't match keeps the - * label in the observation, where an undeclared type can still be reported. - * - * Each shape is the label's declared uniqueness key, plus properties the node fragment writes - * unconditionally where those add discrimination. Shapes are kept minimal: one demanding a property - * dice doesn't always write would make dice's own nodes look foreign and bring back the - * self-reporting case above. - * - * `DiceBookkeepingShapeTest` pins that every label in [CollectorTraceSchema.LABELS] and - * [MetamodelSchema.LABELS] has an entry here, so adding a node label to either store can't quietly - * skip this map. - */ -val DICE_BOOKKEEPING_LABEL_SHAPES: Map> = mapOf( - // Core persistence: propositions and their mentions, provenance, chunk history. - "Proposition" to listOf("id", "contextId", "text"), - "Mention" to listOf("id", "span", "type", "role"), - "Source" to listOf("key"), - "ProcessedChunk" to listOf("id"), - - // Lineage records. - "ProjectionRecord" to listOf("propositionId", "runId", "target"), - "CollectorRecord" to listOf("propositionId", "runId"), - "CollectorRun" to listOf("runId"), - - // Collector trace. Uniqueness keys only — the trace store writes these through several - // statements, and a stricter shape would risk calling its own half-written run foreign. - "CollectorTraceRun" to listOf("runId"), - "CollectorCandidateEdge" to listOf("id"), - "CollectorSignalScore" to listOf("id"), - "CollectorComponent" to listOf("id"), - "CollectorDecision" to listOf("id"), - "CollectorRetired" to listOf("id"), - - // Metamodel governance. - "MetamodelVersion" to listOf("schemaName", "contentHash"), - "MetamodelSchemaCounter" to listOf("schemaName"), - "MetamodelDriftReport" to listOf("schemaName", "versionHash", "capturedAt", "contextKey"), - "MetamodelDriftReportCounter" to listOf("schemaName"), -) - -/** The bookkeeping label names, for callers that only need the names. */ -val DICE_BOOKKEEPING_LABELS: Set = DICE_BOOKKEEPING_LABEL_SHAPES.keys - /** * Relationship types dice writes for its own bookkeeping, on the same grounds as - * [DICE_BOOKKEEPING_LABEL_SHAPES] and under the same shape rule. + * [DiceOwnedSchema] and under the same ownership rule. * * Dice's bookkeeping extends past node labels. `HAS_MENTION` and `DERIVED_FROM` sit on every * proposition ever stored, so without this set a whole-graph observation reports them as undeclared @@ -103,10 +49,17 @@ val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( * * There are two observation paths, because the database offers no single query that answers both: * - * - **Whole graph** (`contextId == null`) introspects the database's own catalogue, `db.labels()` - * and `db.relationshipTypes()`, and subtracts dice's bookkeeping from both sides. The subtraction - * goes by node shape; see [DICE_BOOKKEEPING_LABEL_SHAPES]. A bookkeeping name the domain is also - * using stays in the observation, so an undeclared type can still be reported. + * - **Whole graph** (`contextId == null`) reads the database's own catalogue, `db.labels()` and + * `db.relationshipTypes()`, and subtracts what dice owns from both sides. Ownership goes by node + * shape; see [DiceOwnedSchema]. A dice label the domain is also using stays in the observation, so + * an undeclared type can still be reported. + * + * It then asks a second question, and reports the answer in its own set: the distinct + * `Mention.type` values on dice's own propositions, across the whole graph, returned as + * [ObservedSchema.mentionTypeNames]. A mention type is what an extractor claimed a span was, and + * it becomes a graph label only when something projects it, so a graph can hold live propositions + * mentioning `Ghost` while `db.labels()` has never heard of it. Reading labels alone left that + * type invisible to every unscoped check, which is what this query fixes. * - **One context** (`contextId != null`) cannot use those procedures: they have no notion of a * context and answer for the whole database. It derives both sides from that context's own data: * - entity types are the distinct `Mention.type` values on that context's propositions; @@ -117,13 +70,26 @@ val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( * in both. An undeclared relationship type present in a context's data is drift in that context * whoever else produced it. * - * The scoped entity side is unfiltered. Bookkeeping exclusions are Neo4j labels, while a mention's - * `type` is a domain type name an extractor produced, so the two live in different namespaces and - * subtracting one from the other would hide real drift from an app governing a type called `Source`. + * Mention types are reported as extraction wrote them, on both paths. Dice's ownership rules cover + * Neo4j labels, while a mention's `type` is a domain type name an extractor produced, so the two + * live in different namespaces and subtracting one from the other would hide real drift from an app + * governing a type called `Source`. + * + * ## Two kinds of name, kept apart + * + * The whole-graph observation answers with labels in [ObservedSchema.entityTypeNames], tagged + * [ObservedSchema.EntityTypeBasis.GRAPH_LABELS], and mention types in + * [ObservedSchema.mentionTypeNames]. The differ then judges each by its own rule: a label against + * every label a declared type carries, so an inherited parent label of a governed type reads as + * declared, and a mention type against declared type names and their declared former names alone. + * Merging the two would have to pick one rule for both, and picking the label rule reopens what the + * mention rule exists to close — a mention typed `Agent` passing under a schema that governs + * `Person` with parent label `Agent` and declares no `Agent` type. An unscoped check and a + * context-scoped one now read mention types the same strict way. * * Two limits follow from working off names and shape: * - * 1. Exclusion is decided per label, not per node. If any node wearing a bookkeeping label fails + * 1. Ownership is decided per label, and never per node. If any node wearing a dice label fails * dice's shape, the whole label stays observed, dice's own nodes included, so a graph mixing a * domain `Source` with dice's own reports `Source` every run until the domain type is declared. * 2. Deciding it costs a scan of dice's own labels on every unscoped observation. Each probe stops @@ -148,14 +114,14 @@ open class DrivineObservedSchemaSource( "CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType" /** - * Bookkeeping labels the domain has also claimed: those carrying at least one node that - * fails dice's shape for them. Whatever this returns stays in the observation. + * Dice labels the domain has also claimed: those carrying at least one node that fails + * dice's shape for them. Whatever this returns stays in the observation. * * One branch per label, each stopping at the first non-conforming node, unioned into a - * single round trip. Label and property names are this file's own compile-time constants; - * nothing caller-derived is assembled in. + * single round trip. Label and property names come from [DiceOwnedSchema], which derives + * them from the storage definitions; nothing caller-derived is assembled in. */ - private val LABELS_CLAIMED_BY_DOMAIN: String = DICE_BOOKKEEPING_LABEL_SHAPES.entries + private val LABELS_CLAIMED_BY_DOMAIN: String = DiceOwnedSchema.NODE_SHAPES.entries .joinToString("\nUNION ALL\n") { (label, shape) -> val notDiceShaped = shape.joinToString(" OR ") { property -> "n.$property IS NULL" } "MATCH (n:$label) WHERE $notDiceShaped RETURN '$label' AS label LIMIT 1" @@ -172,6 +138,32 @@ open class DrivineObservedSchemaSource( RETURN DISTINCT type(r) """.trimIndent() + /** + * Entity types across the whole graph: what dice's own propositions mention, wherever they + * live. + * + * The catalogue this file reads for labels knows nothing about mention types. A type an + * extractor wrote reaches `db.labels()` only if something projected a node for it, so a + * graph can hold active propositions mentioning `Ghost` with no `(:Ghost)` node anywhere, + * and a whole-graph check reading labels alone calls that graph clean. This query asks the + * propositions themselves. + * + * Both ends are held to dice's own shape, so a domain node that happens to wear + * `:Proposition` or `:Mention` contributes nothing: what comes back is the set of types + * dice's own extraction recorded. `m.type` is part of the mention shape, so a mention with + * no type is already excluded by it. + * + * Every proposition counts, whatever its status, which is how the context-scoped query + * reads too. A quarantined proposition still carries the undeclared type it was quarantined + * for, and a check that stopped reporting it would read clean while the data sits there. + */ + private val MENTION_TYPES_IN_GRAPH = """ + MATCH (p:Proposition)-[:HAS_MENTION]->(m:Mention) + WHERE ${DiceOwnedSchema.ownedNodePredicate("p", "Proposition")} + AND ${DiceOwnedSchema.ownedNodePredicate("m", "Mention")} + RETURN DISTINCT m.type + """.trimIndent() + /** * Entity types in one context: what its propositions actually mention. * @@ -245,16 +237,21 @@ open class DrivineObservedSchemaSource( override fun observe(): ObservedSchema = observe(null) private fun observeWholeGraph(): ObservedSchema { - // Subtract only the bookkeeping the domain has not also claimed. A name both dice and the + // Subtract only the storage the domain has not also claimed. A name both dice and the // domain use counts as the domain's here, so it stays observable as drift. - val hiddenLabels = DICE_BOOKKEEPING_LABELS - queryStrings(LABELS_CLAIMED_BY_DOMAIN) + val hiddenLabels = DiceOwnedSchema.LABELS - queryStrings(LABELS_CLAIMED_BY_DOMAIN) val hiddenRelationshipTypes = DICE_BOOKKEEPING_RELATIONSHIP_TYPES - queryStrings(RELATIONSHIP_TYPES_CLAIMED_BY_DOMAIN) + // Two kinds of entity name, in two sets: the labels the graph reports, and the types dice's + // propositions were extracted with. The mention side needs no subtraction, since the query + // that produced it already asked dice's own propositions, and it stays out of the label set + // so the differ can hold it to the mention rule; see the class doc. return ObservedSchema( entityTypeNames = queryStrings(ALL_LABELS) - hiddenLabels, relationshipTypeNames = queryStrings(ALL_RELATIONSHIP_TYPES) - hiddenRelationshipTypes, capturedAt = clock.instant(), entityTypeBasis = ObservedSchema.EntityTypeBasis.GRAPH_LABELS, + mentionTypeNames = queryStrings(MENTION_TYPES_IN_GRAPH), ) } diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineProjectionRecordStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineProjectionRecordStore.kt index 36632352..bfc9cb4c 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineProjectionRecordStore.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineProjectionRecordStore.kt @@ -41,8 +41,10 @@ class DrivineProjectionRecordStore( private val logger = LoggerFactory.getLogger(DrivineProjectionRecordStore::class.java) /** - * Upsert the record on its natural key (proposition + run + target) so a replayed projection - * outcome updates in place rather than piling up duplicate nodes. + * Upsert the record on the natural key [LineageSchema] declares for it (proposition + run + + * target), so a replayed projection outcome updates in place and no duplicate node piles up. + * The pattern is built from that key, which keeps the write and the uniqueness constraint + * protecting it on one definition. */ @Transactional override fun record(record: ProjectionRecord) { @@ -50,7 +52,7 @@ class DrivineProjectionRecordStore( persistenceManager.execute( QuerySpecification.withStatement( """ - MERGE (n:ProjectionRecord {propositionId: ${'$'}propositionId, runId: ${'$'}runId, target: ${'$'}target}) + MERGE ${LineageSchema.mergePattern("n", LineageSchema.PROJECTION_RECORD)} SET n.targetRef = ${'$'}targetRef, n.lifecycle = ${'$'}lifecycle, n.at = ${'$'}at, diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt new file mode 100644 index 00000000..6cf88714 --- /dev/null +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import org.drivine.schema.SchemaItemSpec +import org.drivine.schema.UniquenessConstraintSpec + +/** + * The node labels the lineage record stores write, and the natural key each one is upserted on, as + * plain data. + * + * Three things read this and have to agree. [DrivineProjectionRecordStore] and + * [DrivineCollectorRecordStore] build their MERGE patterns from [mergePattern], so a record is + * always upserted on the key named here. A host, and the integration-test harness, declares [specs] + * so those MERGEs are race-free: a MERGE on an unconstrained key lets concurrent writers all miss + * the match, all create, and fill the lineage table with duplicates. [DiceOwnedSchema] reads the + * same keys to work out which `(:ProjectionRecord)` and `(:CollectorRecord)` nodes in a graph are + * dice's own. + * + * Keeping all three off one map is what stops them drifting apart. A key that appears in a store's + * Cypher and nowhere else can lose its constraint, or stop matching the shape an observation + * recognises, with nothing failing to say so. + */ +object LineageSchema { + + /** One node per projection outcome. */ + const val PROJECTION_RECORD: String = "ProjectionRecord" + + /** One node per collector decision about a proposition. */ + const val COLLECTOR_RECORD: String = "CollectorRecord" + + /** One node per collector run. */ + const val COLLECTOR_RUN: String = "CollectorRun" + + /** + * The properties each record is keyed on: the values that decide whether a write updates an + * existing node or creates one. + */ + val NATURAL_KEYS: Map> = mapOf( + PROJECTION_RECORD to listOf("propositionId", "runId", "target"), + COLLECTOR_RECORD to listOf("propositionId", "runId"), + COLLECTOR_RUN to listOf("runId"), + ) + + /** Every node label the lineage stores write, for test cleanup and for drift exclusion. */ + val LABELS: List = NATURAL_KEYS.keys.toList() + + /** A uniqueness constraint per natural key, which is what makes the stores' MERGEs race-free. */ + fun specs(): List = NATURAL_KEYS.map { (label, key) -> + UniquenessConstraintSpec(label = label, properties = key) + } + + /** + * The `(alias:Label {property: $property, ...})` pattern a store MERGEs on. + * + * Label and property names are this object's own compile-time constants, and every value is a + * bound parameter named after its property, so nothing caller-derived is assembled into Cypher. + * The parameter names line up with the bind maps in `LineageRowMappers`. + * + * @param alias The variable the pattern binds the node to. + * @param label One of the labels above. + * @return The pattern text, ready to follow a `MERGE`. + */ + fun mergePattern(alias: String, label: String): String { + val key = requireNotNull(NATURAL_KEYS[label]) { "'$label' is no lineage label" } + return "($alias:$label {" + key.joinToString(", ") { "$it: ${'$'}$it" } + "})" + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt index 449832cd..b94a0bf8 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/AbstractMetamodelVersionStoreContractTest.kt @@ -17,14 +17,16 @@ package com.embabel.dice.storage import com.embabel.dice.metamodel.MetamodelVersion import com.embabel.dice.metamodel.MetamodelVersionStore +import com.embabel.dice.metamodel.SweptBaselineStore import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test /** * Cross-backend contract for [MetamodelVersionStore]: the upsert, history ordering, keyed lookup, - * and schema isolation. Each subclass supplies a store and inherits the whole suite, so a backend - * that disagrees with the in-memory reference fails at authoring time. + * schema isolation, and the swept baseline a [SweptBaselineStore] tracks alongside it. Each + * subclass supplies a store and inherits the whole suite, so a backend that disagrees with the + * in-memory reference fails at authoring time. * * The rules here matter because the drift check re-stamps its schema on every pass. A store that * treated each of those re-stamps as a new record would fill the history with copies of one version, @@ -32,8 +34,14 @@ import org.junit.jupiter.api.Test */ abstract class AbstractMetamodelVersionStoreContractTest { - /** A store holding nothing for the schema names below. */ - protected abstract fun store(): MetamodelVersionStore + /** + * A store holding nothing for the schema names below. + * + * Typed as [SweptBaselineStore], since the swept-baseline half of this suite is a promise only + * a store that tracks the pointer durably can make. A backend that cannot keeps the plain + * [MetamodelVersionStore] contract and has no business inheriting these tests. + */ + protected abstract fun store(): SweptBaselineStore /** A stamp of one entity type. */ private fun version( @@ -140,16 +148,16 @@ abstract class AbstractMetamodelVersionStoreContractTest { // ---- the swept baseline, tracked apart from ordinary write order ---- // - // A store that doesn't override `sweptVersion`/`markSwept` inherits the interface default, - // forwarding to `latestVersion`/`saveVersion` — see `MetamodelVersionStore.sweptVersion`'s doc - // for why that reopens the exact bug `DefaultDriftCheckRunner` relies on this pointer to close. - // Three of the four tests below fail against that default: the null-until-swept case, the - // independence-from-a-later-new-stamp case, and the independence-from-a-later-re-save case. The - // middle test, `markSwept moves the swept baseline to that version`, passes against the - // forwarding default too — a schema with exactly one saved version has that version as both its - // `latestVersion` and (via `markSwept`'s forwarding to `saveVersion`) its only candidate, so the - // default answers correctly by coincidence on a single-version schema. It stays in the suite as a - // positive check on the real behavior; it just isn't the one that catches a non-overriding store. + // `SweptBaselineStore` gives neither method a default body, which is what these four hold a + // store to. A forwarding default answering from `latestVersion`/`saveVersion` would reopen the + // exact bug `DefaultDriftCheckRunner` relies on this pointer to close — see + // `SweptBaselineStore.sweptVersion`'s doc. Three of the four tests below catch a store that + // answers that way: the null-until-swept case, the independence-from-a-later-new-stamp case, + // and the independence-from-a-later-re-save case. The middle test, `markSwept moves the swept + // baseline to that version`, would pass against a forwarding store too — a schema with exactly + // one saved version has that version as both its `latestVersion` and its only swept candidate, + // so write order answers correctly there by coincidence. It stays in the suite as a positive + // check on the real behavior. @Test fun `sweptVersion is null until markSwept is called`() { diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt new file mode 100644 index 00000000..6a3d33ec --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt @@ -0,0 +1,98 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +/** + * Unit tests for [DiceOwnedSchema]: no database, just the shapes it reads out of the storage + * definitions. + * + * These pin the derivation itself. `DrivineObservedSchemaSourceIntegrationTest` pins the thing that + * matters downstream — nodes dice really wrote stay out of an observation, and domain nodes sharing + * a label stay in it — which is what would catch a shape that drifted away from the writers. + */ +class DiceOwnedSchemaTest { + + @Test + fun `a source's shape holds every property dice writes on one`() { + // The reviewer's case in one line: `key` alone is dice's uniqueness key, and a host is free + // to key its own Source type the same way. `kind` is what dice also always writes. + assertEquals(listOf("key", "kind"), DiceOwnedSchema.NODE_SHAPES["Source"]) + } + + @Test + fun `a mention's shape is what an extractor always records`() { + assertEquals(listOf("id", "span", "type", "role"), DiceOwnedSchema.NODE_SHAPES["Mention"]) + } + + @Test + fun `a property dice can leave out stays out of the shape`() { + val proposition = DiceOwnedSchema.NODE_SHAPES.getValue("Proposition") + + assertEquals(listOf("id", "contextId", "text", "confidence", "created"), proposition) + assertFalse(proposition.contains("status"), "status carries a default, so an older node can lack it") + assertFalse(proposition.contains("embedding"), "embedding is nullable") + assertFalse(proposition.contains("metadata"), "the property bag is spread over metadata.") + assertFalse(proposition.contains("grounding"), "an empty list leaves no property behind") + } + + @Test + fun `a Cypher-backed store's shape is every property its keys name`() { + // These stores MERGE on their natural keys, so a node they created carries all of them. The + // version node takes its sequence in the same statement that creates it. + assertEquals( + listOf("schemaName", "contentHash", "sequence"), + DiceOwnedSchema.NODE_SHAPES["MetamodelVersion"], + ) + assertEquals( + listOf("propositionId", "runId", "target"), + DiceOwnedSchema.NODE_SHAPES["ProjectionRecord"], + ) + assertEquals(listOf("runId"), DiceOwnedSchema.NODE_SHAPES["CollectorTraceRun"]) + } + + @Test + fun `every label a dice store declares has a shape`() { + // The derivation has to reach all three schema objects. A label declared by a store and + // missing here would be reported as domain drift on every whole-graph check. + (CollectorTraceSchema.LABELS + MetamodelSchema.LABELS + LineageSchema.LABELS).forEach { label -> + assertTrue( + DiceOwnedSchema.NODE_SHAPES.containsKey(label), + "'$label' is written by a dice store and carries no ownership shape", + ) + } + } + + @Test + fun `the ownership predicate asks for every property of the shape`() { + assertEquals( + "s.key IS NOT NULL AND s.kind IS NOT NULL", + DiceOwnedSchema.ownedNodePredicate("s", "Source"), + ) + } + + @Test + fun `a label dice never writes has no ownership predicate`() { + val thrown = assertThrows { DiceOwnedSchema.ownedNodePredicate("n", "Ghost") } + + assertTrue(thrown.message!!.contains("Ghost"), "got ${thrown.message}") + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt index d792ec12..a1e457f3 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt @@ -20,9 +20,11 @@ import com.embabel.dice.common.DiceMetadataKeys import com.embabel.dice.metamodel.DeclaredSchema import com.embabel.dice.metamodel.DeclaredSchemaSource import com.embabel.dice.metamodel.DriftCheckRunner +import com.embabel.dice.metamodel.DriftSweepCapable import com.embabel.dice.metamodel.MetamodelVersion import com.embabel.dice.metamodel.support.DefaultDriftCheckRunner import com.embabel.dice.metamodel.support.MentionTypeDriftQuarantinePolicy +import com.embabel.dice.metamodel.support.PropositionStoreDriftSweep import com.embabel.dice.metamodel.support.StructuralMetamodelDiffer import com.embabel.dice.proposition.EntityMention import com.embabel.dice.proposition.MentionRole @@ -32,6 +34,7 @@ import org.drivine.manager.PersistenceManager import org.drivine.query.QuerySpecification import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -42,14 +45,17 @@ import org.springframework.test.context.DynamicPropertySource /** * The whole drift check end to end, on real Drivine stores against a Neo4j testcontainer: - * `DefaultDriftCheckRunner` wired to [DrivineMetamodelVersionStore], [DrivineObservedSchemaSource], - * [DrivineDriftReportStore] and [DrivinePropositionRepository], with the real differ and the real - * quarantine policy. + * `DefaultDriftCheckRunner` wired to [DrivineMetamodelVersionStore], [DrivineObservedSchemaSource] + * and [DrivineDriftReportStore], with the real differ. + * + * A check reports and moves nothing, so acting on what it found is a second, deliberate step: these + * tests sweep through [PropositionStoreDriftSweep] over [DrivinePropositionRepository], with the + * real quarantine policy, exactly the way a host would. * * The unit tests in `dice-metamodel` pin the runner's sequencing against fakes. What only a database - * can answer is whether the three persistent pieces line up: a report written by one store names a - * hash the other store can resolve, and the proposition the policy flagged comes back out of the - * graph flagged. + * can answer is whether the persistent pieces line up: a report written by one store names a hash + * the other store can resolve, and the proposition the policy flagged comes back out of the graph + * flagged. */ @SpringBootTest(classes = [TestApplication::class]) class DrivineDriftCheckIntegrationTest { @@ -101,11 +107,40 @@ class DrivineDriftCheckIntegrationTest { differ = differ, metamodelDiffer = differ, driftReportStore = reportStore, - quarantinePolicy = MentionTypeDriftQuarantinePolicy(), - propositionStore = repository, ) } + /** + * A second declaration, governing one type that carries a parent label. Every `Person` node + * carries `Agent` too, and nothing declares `Agent` a type of its own, which is the pair of facts + * an unscoped check has to keep apart. + */ + private val hierarchyVersion = MetamodelVersion( + schemaName = schemaName, + entityTypeNames = listOf("Person"), + entityTypeLabels = mapOf("Person" to setOf("Person", "Agent")), + entityTypeProperties = mapOf("Person" to emptySet()), + relationshipNames = emptyList(), + ) + + private val hierarchyRunner: DriftCheckRunner by lazy { + DefaultDriftCheckRunner( + declaredSchemaSource = DeclaredSchemaSource { + DeclaredSchema(version = hierarchyVersion, relationshipTypeNames = emptySet()) + }, + versionStore = versionStore, + observedSchemaSource = observedSchemaSource, + differ = differ, + metamodelDiffer = differ, + driftReportStore = reportStore, + ) + } + + /** The deliberate half: what a host calls once it has read a check's report and decided. */ + private val sweep: DriftSweepCapable by lazy { PropositionStoreDriftSweep(repository) } + + private val policy = MentionTypeDriftQuarantinePolicy() + @AfterEach fun cleanUp() { repository.clearAll() @@ -113,7 +148,7 @@ class DrivineDriftCheckIntegrationTest { } @Test - fun `an undeclared mention type is reported, resolvable, and quarantined`() { + fun `an undeclared mention type is reported, resolvable, and quarantined by a sweep`() { val stranded = repository.save( Proposition( contextId = contextId, @@ -123,11 +158,10 @@ class DrivineDriftCheckIntegrationTest { ), ) - val result = runner.run(dryRun = false, contextId = contextId) + val result = runner.run(contextId) // 1. The check saw the undeclared type and nothing else. assertEquals(setOf("Ghost"), result.report.driftedEntityTypes) - assertEquals(1, result.quarantinedCount) // 2. The report is really in the graph, under the context it was scoped to. val persisted = reportStore.driftReportsInContext(schemaName, contextId, limit = 10) @@ -143,7 +177,12 @@ class DrivineDriftCheckIntegrationTest { assertNotNull(resolved, "a persisted report named a version hash nothing recorded") assertEquals(declaredVersion, resolved) - // 4. The stranded proposition came back out of the graph flagged, carrying a readable + // 4. The check moved nothing, so a host sweeps the context it decided to reconcile. The + // sweep evaluates the same merged comparison the report showed. + val swept = sweep.sweep(result.quarantineDiff, policy, contextId) + assertEquals(listOf(stranded.id), swept.quarantined.map { it.proposition.id }) + + // 5. The stranded proposition came back out of the graph flagged, carrying a readable // quarantine reason. val reloaded = repository.findById(stranded.id) assertNotNull(reloaded) @@ -154,7 +193,7 @@ class DrivineDriftCheckIntegrationTest { } @Test - fun `a dry run records the same report and touches no proposition`() { + fun `a check records its report and touches no proposition`() { val untouched = repository.save( Proposition( contextId = contextId, @@ -164,12 +203,15 @@ class DrivineDriftCheckIntegrationTest { ), ) - val result = runner.run(dryRun = true, contextId = contextId) + val result = runner.run(contextId) assertEquals(setOf("Ghost"), result.report.driftedEntityTypes) - assertEquals(0, result.quarantinedCount) assertEquals(1, reportStore.driftReportsInContext(schemaName, contextId, limit = 10).size) - assertEquals(PropositionStatus.ACTIVE, repository.findById(untouched.id)!!.status) + assertEquals( + PropositionStatus.ACTIVE, + repository.findById(untouched.id)!!.status, + "a check reports what it found and leaves every proposition where it was", + ) } @Test @@ -185,10 +227,13 @@ class DrivineDriftCheckIntegrationTest { ), ) - val result = runner.run(dryRun = false, contextId = contextId) + val result = runner.run(contextId) assertTrue(result.report.driftedEntityTypes.isEmpty(), "got ${result.report.driftedEntityTypes}") - assertEquals(0, result.quarantinedCount) + assertTrue( + sweep.sweep(result.quarantineDiff, policy, contextId).quarantined.isEmpty(), + "a clean context gives a sweep nothing to do", + ) assertEquals(listOf(result.report), reportStore.driftReportsInContext(schemaName, contextId, limit = 10)) } @@ -203,8 +248,8 @@ class DrivineDriftCheckIntegrationTest { ), ) - val first = runner.run(dryRun = true, contextId = contextId) - val second = runner.run(dryRun = true, contextId = contextId) + val first = runner.run(contextId) + val second = runner.run(contextId) val history = reportStore.driftReportsInContext(schemaName, contextId, limit = 10) assertEquals(listOf(second.report, first.report), history, "a drift log accumulates; it is not a gauge") @@ -221,14 +266,113 @@ class DrivineDriftCheckIntegrationTest { ), ) - val result = runner.run(dryRun = false, contextId = contextId) + val result = runner.run(contextId) assertTrue(result.report.driftedEntityTypes.isEmpty(), "got ${result.report.driftedEntityTypes}") - assertEquals(0, result.quarantinedCount) + sweep.sweep(result.quarantineDiff, policy, contextId) assertEquals( PropositionStatus.ACTIVE, repository.findByContextId(ContextId("elsewhere")).single().status, "a check scoped to one context must not be able to reach another's data", ) } + + @Test + fun `an unscoped check reports a mention type nothing ever projected, and a sweep quarantines it`() { + // The whole-graph path reads the database's label catalogue, and a mention type reaches that + // catalogue only once something projects a node for it. An extraction that recorded `Ghost` + // and produced no `(:Ghost)` node left the graph looking clean to every unscoped check while + // a live proposition carried the undeclared type. + val stranded = repository.save( + Proposition( + contextId = contextId, + text = "The ghost haunts the manor", + mentions = listOf(EntityMention(span = "the ghost", type = "Ghost", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + assertEquals(PropositionStatus.ACTIVE, repository.findById(stranded.id)!!.status) + assertFalse( + rawLabels().contains("Ghost"), + "precondition: no typed graph projection for Ghost, but the catalogue held ${rawLabels()}", + ) + + val result = runner.run() + + assertTrue( + result.report.driftedEntityTypes.contains("Ghost"), + "the unscoped check missed a type only the propositions know about; got " + + "${result.report.driftedEntityTypes}", + ) + assertEquals( + listOf(result.report), + reportStore.globalDriftReports(schemaName, limit = 10), + "an unscoped check records a whole-graph report", + ) + + // The quarantine half is the host's deliberate step, on the context holding the data. + val swept = sweep.sweep(result.quarantineDiff, policy, contextId) + + assertEquals(listOf(stranded.id), swept.quarantined.map { it.proposition.id }) + assertEquals(PropositionStatus.STALE, repository.findById(stranded.id)!!.status) + } + + @Test + fun `a projected node's parent label is no drift on an unscoped check`() { + // One half of the pair. The graph reports `Agent` as a label, because every governed + // `Person` node carries its whole hierarchy, and the declaration says so. + repository.save( + Proposition( + contextId = contextId, + text = "Ada is a person", + mentions = listOf(EntityMention(span = "Ada", type = "Person", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + projectNodeCarryingHierarchy() + + val result = hierarchyRunner.run() + + assertTrue( + result.report.driftedEntityTypes.isEmpty(), + "a governed type's own hierarchy label read as drift; got ${result.report.driftedEntityTypes}", + ) + } + + @Test + fun `a mention typed as a parent label is drift on an unscoped check`() { + // The other half, on a graph holding the same node. Only the mention differs: this one claims + // to BE an `Agent`, a type nothing declared. Reading both kinds of name under the label rule + // let that claim ride the governed type's hierarchy through an unscoped check. + val stranded = repository.save( + Proposition( + contextId = contextId, + text = "Ada answers for the estate", + mentions = listOf(EntityMention(span = "Ada", type = "Agent", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + projectNodeCarryingHierarchy() + + val result = hierarchyRunner.run() + + assertEquals(setOf("Agent"), result.report.driftedEntityTypes) + val swept = sweep.sweep(result.quarantineDiff, policy, contextId) + assertEquals(listOf(stranded.id), swept.quarantined.map { it.proposition.id }) + assertEquals(PropositionStatus.STALE, repository.findById(stranded.id)!!.status) + } + + /** A projected entity node carrying a governed `Person`'s whole label hierarchy. */ + private fun projectNodeCarryingHierarchy() { + persistenceManager.execute(QuerySpecification.withStatement("CREATE (:Person:Agent {id: 'e-ada'})")) + } + + /** What the database's own catalogue reports, before an observation subtracts anything from it. */ + private fun rawLabels(): Set = persistenceManager + .query( + QuerySpecification.withStatement("CALL db.labels() YIELD label RETURN label") + .transform(String::class.java), + ) + .filterNotNull() + .toSet() } diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStoreContractIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStoreContractIntegrationTest.kt index a693d4f3..3610237e 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStoreContractIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineMetamodelVersionStoreContractIntegrationTest.kt @@ -15,7 +15,7 @@ */ package com.embabel.dice.storage -import com.embabel.dice.metamodel.MetamodelVersionStore +import com.embabel.dice.metamodel.SweptBaselineStore import org.drivine.manager.PersistenceManager import org.drivine.query.QuerySpecification import org.junit.jupiter.api.AfterEach @@ -48,7 +48,7 @@ class DrivineMetamodelVersionStoreContractIntegrationTest : AbstractMetamodelVer @Autowired private lateinit var persistenceManager: PersistenceManager - override fun store(): MetamodelVersionStore = graphStore + override fun store(): SweptBaselineStore = graphStore @AfterEach fun cleanUp() { diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt index ba7aefca..877c67a5 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt @@ -16,9 +16,17 @@ package com.embabel.dice.storage import com.embabel.agent.core.ContextId +import com.embabel.dice.incremental.BookmarkKey +import com.embabel.dice.incremental.HashKey +import com.embabel.dice.incremental.ProcessedChunkRecord import com.embabel.dice.metamodel.DriftReport import com.embabel.dice.metamodel.MetamodelVersion import com.embabel.dice.metamodel.ObservedSchema +import com.embabel.dice.proposition.EntityMention +import com.embabel.dice.proposition.MentionRole +import com.embabel.dice.proposition.Proposition +import com.embabel.dice.provenance.ProvenanceEntry +import com.embabel.dice.provenance.UriLocator import org.drivine.manager.PersistenceManager import org.drivine.query.QuerySpecification import org.junit.jupiter.api.AfterEach @@ -60,6 +68,12 @@ class DrivineObservedSchemaSourceIntegrationTest { @Autowired private lateinit var persistenceManager: PersistenceManager + @Autowired + private lateinit var repository: DrivinePropositionRepository + + @Autowired + private lateinit var chunkHistoryStore: DrivineChunkHistoryStore + private val tenantA = ContextId("tenant-a") private val tenantB = ContextId("tenant-b") @@ -147,6 +161,53 @@ class DrivineObservedSchemaSourceIntegrationTest { assertEquals(ObservedSchema.EntityTypeBasis.GRAPH_LABELS, observed.entityTypeBasis) } + @Test + fun `whole-graph observation reports a mention type that has no graph label`() { + // A mention type reaches `db.labels()` only once something projects a node for it, so the + // label catalogue alone answers "clean" for a graph whose propositions carry an undeclared + // type. The observation asks the propositions as well, and answers in its own set. + writeProposition("p-a", tenantA) + writeMention("p-a", "m-a", type = "Ghost") + assertFalse(rawLabels().contains("Ghost"), "precondition: no (:Ghost) node exists") + + val observed = source.observe() + + assertTrue( + observed.mentionTypeNames.contains("Ghost"), + "an undeclared mention type stayed invisible to the whole-graph check; got " + + "${observed.mentionTypeNames}", + ) + assertFalse( + observed.entityTypeNames.contains("Ghost"), + "mention types stay out of the label set, so each side keeps its own comparison rule; " + + "got ${observed.entityTypeNames}", + ) + assertEquals(ObservedSchema.EntityTypeBasis.GRAPH_LABELS, observed.entityTypeBasis) + } + + @Test + fun `whole-graph mention types come from dice's own propositions`() { + // A domain node wearing `:Proposition` is somebody else's record, and whatever it calls its + // mentions is its own business. The label half already reports both labels as domain data. + persistenceManager.execute( + QuerySpecification.withStatement( + "CREATE (:Proposition {headline: 'not ours'})-[:HAS_MENTION]->(:Mention {type: 'Impostor'})", + ), + ) + + val observed = source.observe() + + assertFalse( + observed.mentionTypeNames.contains("Impostor"), + "a mention type off a node dice never wrote reached the observation; got " + + "${observed.mentionTypeNames}", + ) + assertTrue( + observed.entityTypeNames.containsAll(setOf("Proposition", "Mention")), + "the domain claimed both labels, so both stay observable; got ${observed.entityTypeNames}", + ) + } + @Test fun `an observation is stamped with the instant it was taken`() { val before = Instant.now() @@ -162,8 +223,8 @@ class DrivineObservedSchemaSourceIntegrationTest { fun `dice's own storage labels are never reported as domain drift`() { writeProposition("p-a", tenantA) writeMention("p-a", "m-a", type = "Person") - persistenceManager.execute(QuerySpecification.withStatement("CREATE (:Source {key: 'src'})")) - persistenceManager.execute(QuerySpecification.withStatement("CREATE (:ProcessedChunk {id: 'chunk'})")) + writeSource("src") + writeProcessedChunk("chunk") // Without this precondition the test would pass on an empty observation. It pins that the // database is reporting these labels, so the exclusion is what keeps them out. assertTrue( @@ -174,8 +235,8 @@ class DrivineObservedSchemaSourceIntegrationTest { val observed = source.observe() assertTrue( - observed.entityTypeNames.none { it in DICE_BOOKKEEPING_LABELS }, - "bookkeeping leaked into the observation: ${observed.entityTypeNames intersect DICE_BOOKKEEPING_LABELS}", + observed.entityTypeNames.none { it in DiceOwnedSchema.LABELS }, + "bookkeeping leaked into the observation: ${observed.entityTypeNames intersect DiceOwnedSchema.LABELS}", ) } @@ -220,7 +281,7 @@ class DrivineObservedSchemaSourceIntegrationTest { writeMention("p-a", "m-a", type = "Person") persistenceManager.execute( QuerySpecification.withStatement( - "MATCH (p:Proposition {id: 'p-a'}) CREATE (p)-[:DERIVED_FROM]->(:Source {key: 'src'})", + "MATCH (p:Proposition {id: 'p-a'}) CREATE (p)-[:DERIVED_FROM]->(:Source {key: 'src', kind: 'uri'})", ), ) assertTrue( @@ -261,8 +322,8 @@ class DrivineObservedSchemaSourceIntegrationTest { // The other half of the shape test: dice's own conforming nodes still get excluded. writeProposition("p-a", tenantA) writeMention("p-a", "m-a", type = "Person") - persistenceManager.execute(QuerySpecification.withStatement("CREATE (:Source {key: 'src-1'})")) - persistenceManager.execute(QuerySpecification.withStatement("CREATE (:ProcessedChunk {id: 'chunk-1'})")) + writeSource("src-1") + writeProcessedChunk("chunk-1") val observed = source.observe() @@ -274,6 +335,64 @@ class DrivineObservedSchemaSourceIntegrationTest { } } + @Test + fun `a domain Source node keyed the way dice keys its own is still observed`() { + // The overlap a key-only rule gets wrong: a host's own `Source` type, carrying a property + // called `key`, which is exactly dice's uniqueness key for the label. Ownership is decided + // on everything dice writes, and dice writes `kind` on every source of its own, so this node + // stays the domain's and the type stays reportable. + persistenceManager.execute( + QuerySpecification.withStatement("CREATE (:Source {key: 'acme-crm', displayName: 'Acme CRM'})"), + ) + writeSource("dice-own-src") + + val observed = source.observe() + + assertTrue( + observed.entityTypeNames.contains("Source"), + "a domain Source carrying only a key is domain data; got ${observed.entityTypeNames}", + ) + } + + @Test + fun `nodes dice really wrote match the shapes derived from its own schema`() { + // The derivation is worth only as much as its agreement with the writers, so this one goes + // through the real repository and the real chunk-history store. A shape demanding a property + // dice sometimes leaves out would show up here as dice reporting its own storage as drift. + repository.save( + Proposition( + contextId = tenantA, + text = "Ada wrote the first algorithm", + mentions = listOf(EntityMention(span = "Ada", type = "Person", role = MentionRole.SUBJECT)), + provenanceEntries = listOf(ProvenanceEntry(locator = UriLocator("https://example.com/ada"))), + confidence = 0.9, + ), + ) + chunkHistoryStore.recordProcessed( + ProcessedChunkRecord( + bookmarkKey = BookmarkKey(tenantA, "source-1"), + hashKey = HashKey(tenantA, "content-hash-1"), + startIndex = 0, + endIndex = 10, + processedAt = Instant.parse("2026-01-01T00:00:00Z"), + ), + ) + val written = listOf("Proposition", "Mention", "Source", "ProcessedChunk") + assertTrue( + rawLabels().containsAll(written.toSet()), + "precondition: the raw catalogue must hold what dice just wrote, but was ${rawLabels()}", + ) + + val observed = source.observe() + + written.forEach { label -> + assertFalse( + observed.entityTypeNames.contains(label), + "dice reported its own '$label' nodes as drift; got ${observed.entityTypeNames}", + ) + } + } + @Test fun `a domain relationship sharing a bookkeeping type's name is still observed`() { // Same rule on the relationship side, decided by the marker the scoped path uses: an edge @@ -292,13 +411,14 @@ class DrivineObservedSchemaSourceIntegrationTest { } @Test - fun `every label the trace and metamodel stores write has a shape entry`() { - // Keeps the shape map in step with the schema objects: a new node label added to either - // store without a shape here would stop being excluded. - (CollectorTraceSchema.LABELS + MetamodelSchema.LABELS).forEach { label -> + fun `every label a dice store declares has an ownership shape`() { + // The shapes are derived from these same schema objects, so this is a pin on the derivation + // reaching all of them: a label declared by a store and missing here would stop being + // excluded. + (CollectorTraceSchema.LABELS + MetamodelSchema.LABELS + LineageSchema.LABELS).forEach { label -> assertTrue( - DICE_BOOKKEEPING_LABEL_SHAPES.containsKey(label), - "'$label' is written by a dice store but has no bookkeeping shape", + DiceOwnedSchema.NODE_SHAPES.containsKey(label), + "'$label' is written by a dice store and carries no ownership shape", ) } } @@ -306,14 +426,42 @@ class DrivineObservedSchemaSourceIntegrationTest { // ---- helpers ---- /** - * A proposition node carrying dice's full shape, which is how the observer recognises it as - * dice's own rather than a domain node sharing the label. + * A proposition node carrying every property dice's own writer always writes, which is how the + * observer recognises it as dice's own where a domain node shares the label. The list is + * [DiceOwnedSchema.NODE_SHAPES] for `Proposition`, read off `PropositionNode`'s required + * constructor parameters. */ private fun writeProposition(id: String, contextId: ContextId) { persistenceManager.execute( QuerySpecification.withStatement( - "CREATE (:Proposition {id: \$id, contextId: \$contextId, text: \$text})", - ).bind(mapOf("id" to id, "contextId" to contextId.value, "text" to "a fact about $id")), + "CREATE (:Proposition {id: \$id, contextId: \$contextId, text: \$text, " + + "confidence: \$confidence, created: \$created})", + ).bind( + mapOf( + "id" to id, + "contextId" to contextId.value, + "text" to "a fact about $id", + "confidence" to 0.9, + "created" to Instant.parse("2026-01-01T00:00:00Z").toString(), + ), + ), + ) + } + + /** Likewise a source node: dice writes `key` and `kind` on every one of its own. */ + private fun writeSource(key: String) { + persistenceManager.execute( + QuerySpecification.withStatement("CREATE (:Source {key: \$key, kind: 'uri'})").bind(mapOf("key" to key)), + ) + } + + /** Likewise a processed chunk, as `DrivineChunkHistoryStore` writes it. */ + private fun writeProcessedChunk(id: String) { + persistenceManager.execute( + QuerySpecification.withStatement( + "CREATE (:ProcessedChunk {id: \$id, contextId: 'tenant-a', contentHash: 'hash', " + + "sourceId: 'src', startIndex: 0, endIndex: 10, processedAt: \$processedAt})", + ).bind(mapOf("id" to id, "processedAt" to Instant.parse("2026-01-01T00:00:00Z").toString())), ) } diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/InMemoryMetamodelVersionStoreContractTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/InMemoryMetamodelVersionStoreContractTest.kt index 2afca2f1..f6467019 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/InMemoryMetamodelVersionStoreContractTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/InMemoryMetamodelVersionStoreContractTest.kt @@ -16,7 +16,7 @@ package com.embabel.dice.storage import com.embabel.dice.metamodel.InMemoryMetamodelVersionStore -import com.embabel.dice.metamodel.MetamodelVersionStore +import com.embabel.dice.metamodel.SweptBaselineStore /** * Runs the [AbstractMetamodelVersionStoreContractTest] suite against the in-memory reference store. @@ -24,5 +24,5 @@ import com.embabel.dice.metamodel.MetamodelVersionStore * the graph IT completes. */ class InMemoryMetamodelVersionStoreContractTest : AbstractMetamodelVersionStoreContractTest() { - override fun store(): MetamodelVersionStore = InMemoryMetamodelVersionStore() + override fun store(): SweptBaselineStore = InMemoryMetamodelVersionStore() } diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/LineageSchemaTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/LineageSchemaTest.kt new file mode 100644 index 00000000..94fb7f9f --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/LineageSchemaTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import org.drivine.schema.UniquenessConstraintSpec +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +/** + * Unit tests for [LineageSchema]: the one place the lineage stores' natural keys live. + * + * The MERGE pattern and the uniqueness constraint are built from the same key here, so these pin + * that they render the key a store is actually upserting on. + * `DrivineLineageRecordStoreIntegrationTest` runs the resulting statements against a database. + */ +class LineageSchemaTest { + + @Test + fun `a merge pattern names every property of the natural key`() { + assertEquals( + "(n:ProjectionRecord {propositionId: \$propositionId, runId: \$runId, target: \$target})", + LineageSchema.mergePattern("n", LineageSchema.PROJECTION_RECORD), + ) + assertEquals( + "(n:CollectorRun {runId: \$runId})", + LineageSchema.mergePattern("n", LineageSchema.COLLECTOR_RUN), + ) + } + + @Test + fun `a label the lineage stores never write has no pattern`() { + assertThrows { LineageSchema.mergePattern("n", "Proposition") } + } + + @Test + fun `every label gets a uniqueness constraint on the key its store merges on`() { + val constraints = LineageSchema.specs().filterIsInstance() + .associate { it.label to it.properties } + + assertEquals(LineageSchema.NATURAL_KEYS, constraints) + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt index 0aed179b..fd30c24c 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt @@ -128,12 +128,12 @@ open class TestApplication { persistenceManager: PersistenceManager, ): DrivineChunkHistoryStore = DrivineChunkHistoryStore(graphObjectManager, persistenceManager) + /** + * The lineage stores MERGE on the keys [LineageSchema] declares, so the constraints come off the + * same list. A host wires it the same way. + */ @Bean - open fun lineageSchema(): SchemaCatalog = SchemaCatalog.of( - UniquenessConstraintSpec(label = "ProjectionRecord", properties = listOf("propositionId", "runId", "target")), - UniquenessConstraintSpec(label = "CollectorRecord", properties = listOf("propositionId", "runId")), - UniquenessConstraintSpec(label = "CollectorRun", property = "runId"), - ) + open fun lineageSchema(): SchemaCatalog = SchemaCatalog.of(LineageSchema.specs()) @Bean open fun projectionRecordStore( diff --git a/docs/design/metamodel-drift.md b/docs/design/metamodel-drift.md index d0e60abd..13f69359 100644 --- a/docs/design/metamodel-drift.md +++ b/docs/design/metamodel-drift.md @@ -630,23 +630,41 @@ back newest first by capture instant, compared to the nanosecond so a `since` wi with a per-schema counter breaking exact ties so a limited page is repeatable. `DrivineObservedSchemaSource` takes the snapshot. Unscoped, it reads the database's own catalogue -(`db.labels()`, `db.relationshipTypes()`); scoped to a context, it derives entity types from that -context's mentions and relationship types from the `sourcePropositions` each projected edge carries. -Either way it subtracts dice's own bookkeeping: the proposition, mention, provenance, lineage, -collector-trace and metamodel node labels, and the `HAS_MENTION`/`DERIVED_FROM`-style edges. That -subtraction is load-bearing. Stamping a version and writing a report both add nodes to the graph the -next check looks at, so without it every run reports the previous run as drift. - -The subtraction goes by node shape, so that a domain type called `Source` stays visible. A -bookkeeping label is excluded only while every node carrying it matches dice's shape for it (dice's -`Source` nodes carry `key`, its governance nodes carry `schemaName`, and so on), and a bookkeeping -relationship type only while no edge of that type carries `sourcePropositions`, the marker the graph -writer stamps on every edge it projects from domain data. The context-scoped query selects on the -same marker. - -Two limits follow. Exclusion is decided per label, not per node, so a graph mixing a domain `Source` -with dice's own reports `Source` every run until the type is declared. And deciding it costs a scan -of dice's own labels on each unscoped observation; context-scoped checks don't pay it. +(`db.labels()`, `db.relationshipTypes()`) **and** asks dice's own propositions for the distinct +`Mention.type` values they carry, reporting those in their own set; scoped to a context, it derives +entity types from that context's mentions and relationship types from the `sourcePropositions` each +projected edge carries. Either way +it subtracts dice's own storage: the proposition, mention, provenance, lineage, collector-trace and +metamodel node labels, and the `HAS_MENTION`/`DERIVED_FROM`-style edges. That subtraction is +load-bearing. Stamping a version and writing a report both add nodes to the graph the next check +looks at, so without it every run reports the previous run as drift. + +The second question the unscoped path asks is there because a mention type reaches `db.labels()` +only once something projects a node for it. An extraction that recorded `Ghost` and produced no +`(:Ghost)` node left a graph full of undeclared data looking clean to every whole-graph check, while +the context-scoped check on the same data reported it. The global query holds both ends to dice's +own shape, so mention types come off dice's extraction records and a domain node wearing +`:Proposition` contributes nothing. + +Ownership goes by node shape, so that a domain type called `Source` stays visible. A dice label is +excluded only while every node carrying it matches dice's shape for it, and a dice relationship type +only while no edge of that type carries `sourcePropositions`, the marker the graph writer stamps on +every edge it projects from domain data. The context-scoped query selects on the same marker. + +The shapes are derived, in `DiceOwnedSchema`, from the storage definitions themselves. A node +fragment's shape is every constructor parameter dice's writer cannot leave out — declared non-null +with no default — so `Source` is `key` **and** `kind`, and a host's own `(:Source {key: ...})` stays +observable where a key-only rule would have hidden it. A Cypher-backed store's shape is the union of +the properties its uniqueness constraints name, which is what it MERGEs on. Adding a label to +`MetamodelSchema`, `CollectorTraceSchema` or `LineageSchema` carries its shape along with it, and an +integration test writes through the real stores and asserts dice never reports its own nodes. + +Three limits follow. Ownership is decided per label, so a graph mixing a domain `Source` with dice's +own reports `Source` every run until the type is declared. Deciding it costs a scan of dice's own +labels on each unscoped observation; context-scoped checks don't pay it. And a domain node carrying +every property dice writes for a label they share is indistinguishable from dice's own; closing that +last case needs an ownership marker written at persistence time, which is a data migration for +existing graphs. `observe` runs its whole set of queries — bookkeeping-exclusion probes included — inside one Neo4j transaction, so the several reads that get assembled into one `ObservedSchema` come from a single @@ -672,6 +690,17 @@ widening. Tagging the two the same way let a mention typed `Agent` conform under governs `Person` with parent `Agent` — an undeclared mention type escaping detection by riding a governed type's parent label — until the observation itself carried which comparison it needs. +An unscoped observation holds both kinds of name, in two sets. `entityTypeNames` carries the labels +and states its basis as before; `ObservedSchema.mentionTypeNames` carries the mention types, needs no +tag, and is always judged by the `MENTION_TYPES` rule. `StructuralMetamodelDiffer.diffAgainstObserved` +compares each set with the declared side its own rule calls for, and unions what drifted. One set for both would have +to pick one rule: picking the label rule reopens exactly what the mention rule closes — a mention +typed `Agent` passing under a schema that governs `Person` with parent label `Agent` and declares no +`Agent` type — and picking the mention rule reports every inherited parent label in the graph as +drift. An unscoped check and a context-scoped one now read mention types the same strict way. Any +source that can only reach a label catalogue leaves `mentionTypeNames` empty, and the comparison is +what it always was. + Hosts declare the constraints these stores need (see `MetamodelSchema`); a MERGE is race-free only under a uniqueness constraint on the key it merges on. @@ -679,11 +708,16 @@ under a uniqueness constraint on the key it merges on. `DriftReportStore` and `ObservedSchemaSource` now have Drivine implementations in `dice-storage`. -`DriftSweepCapable` and `SweptBaselineStore` are contracts with an in-memory reference -implementation and no durable one. Until the graph-backed store implements `SweptBaselineStore`, a -Drivine-backed host gets the graph-truth half of a report and a `null` declared comparison. Until it -implements `DriftSweepCapable`, a host sweeps through `PropositionStoreDriftSweep`, which is correct -and does its filtering in the JVM. +`SweptBaselineStore` now has a durable implementation: `DrivineMetamodelVersionStore` declares it and +keeps the reconciled baseline as `sweptContentHash` on the schema's own `(:MetamodelSchemaCounter)` +node, moved by `markSwept` alone. A Drivine-backed host therefore gets the declared-vs-previous half +of a report as soon as its first sweep completes, and the shared contract suite +(`AbstractMetamodelVersionStoreContractTest`) runs against the graph store and the in-memory +reference alike. + +`DriftSweepCapable` is still a contract with an in-memory reference implementation and no durable +one. Until the graph-backed store implements it, a host sweeps through `PropositionStoreDriftSweep`, +which is correct and does its filtering in the JVM. There is no Spring configuration in `dice-metamodel` or `dice-storage`, so a runner is an ordinary constructor call until the autoconfigure slice assembles one, and nothing sweeps unless a host From be866af19752250af55988452106e4dbd786e03d Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:13:54 -0400 Subject: [PATCH 05/10] Exclude infrastructure labels from whole-graph observation Drivine writes _DrivineSchema for its own schema bookkeeping. DICE's schema definitions cannot derive a label the library owns, so every whole-graph check reported it as drift. Infrastructure labels get their own enumerated exclusion beside the derived catalog, matched by exact name, with a test proving an unknown label still drifts. Also retires the word seam from the module docs and CHANGELOG. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- CHANGELOG.md | 2 +- dice-storage-autoconfigure/AGENTS.md | 6 ++-- .../storage/DrivineObservedSchemaSource.kt | 20 ++++++++++++- ...vineObservedSchemaSourceIntegrationTest.kt | 29 +++++++++++++++++++ docs/design/architecture.md | 4 +-- 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e3ff53b..35abae40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and the consumer PRs that deliver it). - `dice-metamodel` module, first slice of schema versioning: `MetamodelVersion` content-hash stamping with per-type governance selection, the declared-schema - opt-in seam, and the `MetamodelVersionStore` contract. Pure JVM. + opt-in contract, and the `MetamodelVersionStore` contract. Pure JVM. **Compatibility: additive.** New module; no existing API touched. - Declared renames in `dice-metamodel`. **EXPERIMENTAL** (shape may change diff --git a/dice-storage-autoconfigure/AGENTS.md b/dice-storage-autoconfigure/AGENTS.md index cea19827..2015cc49 100644 --- a/dice-storage-autoconfigure/AGENTS.md +++ b/dice-storage-autoconfigure/AGENTS.md @@ -29,7 +29,7 @@ Three rules, and that's the whole mechanism: exception — they carry no `@ConditionalOnMissingBean`, so they're applied whenever the graph backend is active and aren't overridable by a competing bean.) 3. **Graph beans are declared before their in-memory counterparts**, so the flip resolves by - registration order rather than mutually-exclusive conditions. + registration order (with no need for mutually-exclusive conditions). The graph repository and the vector-index schema additionally require `@ConditionalOnBean(Ai::class)` — they need an embedding service, which comes from the embabel-agent `Ai` handle. @@ -43,7 +43,7 @@ by the starter) applies them idempotently on startup — there is no migration r `Source.key`, the composite `(Proposition.contextId, Proposition.text)` dedup backstop, and the range indexes queries filter by (`contextId`, `status`, `level`, `effectiveConfidence`, `Mention.resolvedId`, …). - `lineageRecordSchema` — natural-key uniqueness for `ProjectionRecord`, `CollectorRecord`, and - `CollectorRun`, which is what lets the lineage stores `MERGE` (upsert) instead of duplicating. + `CollectorRun`, which lets the lineage stores `MERGE` (upsert) without duplicating. - `propositionVectorIndexSchema` — the cosine vector index on `Proposition.embedding`, sized to the embedding model's dimension and stamped with the model name as the schema version. Gated behind `embabel.dice.store.vector-index.enabled` (default true). @@ -86,4 +86,4 @@ drift from that annotation and silently break vector search, so they live as con - Changing the embedding model to a different vector dimension requires dropping and recreating the vector index — the schema is applied idempotently but won't resize an existing index. - The decay tick is a no-op when no `DecayManager` is available (resolved lazily), so enabling decay - without a store backend simply does nothing rather than failing. + without a store backend does nothing (it never fails). diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt index 87fd027d..4322a442 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt @@ -43,6 +43,24 @@ val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( "RETIRED_IN", ) +/** + * Node labels written by infrastructure libraries that Dice runs on top of, excluding them from + * whole-graph drift observation. + * + * Infrastructure libraries own these labels and write them as part of their own bookkeeping. + * DICE's schema definitions cannot declare them — the library owns them entirely — so every + * whole-graph observation would report them as undeclared drift on every run if left unexcluded. + * + * Unlike [DICE_BOOKKEEPING_RELATIONSHIP_TYPES], which are decided by shape (a `sourcePropositions` + * marker on edges), these are decided by name. An infrastructure library's bookkeeping is not + * declared anywhere in DICE's schema, so there is no shape to recognize it by. + * + * Currently includes `_DrivineSchema`, written by the Drivine library itself. + */ +val INFRASTRUCTURE_LABELS: Set = setOf( + "_DrivineSchema", +) + /** * Drivine / Neo4j implementation of [ObservedSchemaSource]: asks a live graph what it contains, so a * `DeclaredObservedDiffer` can compare it against what was declared. @@ -247,7 +265,7 @@ open class DrivineObservedSchemaSource( // that produced it already asked dice's own propositions, and it stays out of the label set // so the differ can hold it to the mention rule; see the class doc. return ObservedSchema( - entityTypeNames = queryStrings(ALL_LABELS) - hiddenLabels, + entityTypeNames = queryStrings(ALL_LABELS) - hiddenLabels - INFRASTRUCTURE_LABELS, relationshipTypeNames = queryStrings(ALL_RELATIONSHIP_TYPES) - hiddenRelationshipTypes, capturedAt = clock.instant(), entityTypeBasis = ObservedSchema.EntityTypeBasis.GRAPH_LABELS, diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt index 877c67a5..1c2db9a8 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt @@ -300,6 +300,35 @@ class DrivineObservedSchemaSourceIntegrationTest { // ---- Exclusion by shape, not by label name ---- + @Test + fun `infrastructure labels written by Drivine are never reported as domain drift`() { + // The Drivine library writes the `_DrivineSchema` label as its own bookkeeping and owns it + // entirely. DICE's schema definitions cannot declare it, so without exclusion every + // whole-graph check reports it as drift. This test verifies that the exclusion catches it + // while an unknown label still drifts, proving the exclusion is exact. + persistenceManager.execute( + QuerySpecification.withStatement("CREATE (:_DrivineSchema {something: 'drivine owns this'})"), + ) + persistenceManager.execute( + QuerySpecification.withStatement("CREATE (:UnknownLabel {id: 'drift-marker'})"), + ) + assertTrue( + rawLabels().containsAll(setOf("_DrivineSchema", "UnknownLabel")), + "precondition: the raw catalogue must hold both labels, but was ${rawLabels()}", + ) + + val observed = source.observe() + + assertFalse( + observed.entityTypeNames.contains("_DrivineSchema"), + "Drivine's bookkeeping label must be excluded; got ${observed.entityTypeNames}", + ) + assertTrue( + observed.entityTypeNames.contains("UnknownLabel"), + "unknown labels must still be reported as drift; got ${observed.entityTypeNames}", + ) + } + @Test fun `a domain node that only shares a bookkeeping label's name is still observed`() { // Excluding the name `Source` would hide an app's own undeclared `Source` type from every diff --git a/docs/design/architecture.md b/docs/design/architecture.md index aa92be6c..43cbd944 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -16,7 +16,7 @@ DICE is a multi-module Maven build. Each module's intent, and what it's allowed | `dice-storage-autoconfigure` | Spring Boot autoconfiguration that wires `dice-storage`'s beans (repository, projectors, trust scorer) into a host application. Depends on `dice-storage`. | | `dice-ingestion` | Content-hash dedup ledger and source adapters that sit in front of `PropositionPipeline`, so the same artifact is never extracted twice concurrently. Depends on `dice`. | | `dice-report` | Rationale and structured report generation over propositions and their lineage. Depends on `dice`. | -| `dice-metamodel` | Schema governance: content-hash stamps over the governed part of a `DataDictionary`, the declared-schema seam, the version and drift-report store contracts, diffing, drift checking, and non-destructive quarantine. A leaf over `embabel-agent-api`, with no dependency on `dice`; `dice-storage` implements its store contracts. | +| `dice-metamodel` | Schema governance: content-hash stamps over the governed part of a `DataDictionary`, the declared-schema contract, the version and drift-report store contracts, diffing, drift checking, and non-destructive quarantine. A leaf over `embabel-agent-api`, with no dependency on `dice`; `dice-storage` implements its store contracts. | | `dice-integration-tests` | End-to-end tests exercising the real Neo4j backend and full pipeline across module boundaries. Depends on `dice`, `dice-ingestion`, `dice-report` (and transitively `dice-storage`). Not shipped. | ```mermaid @@ -198,7 +198,7 @@ audit trails survive a restart. See [graph-projection](graph-projection.md). walking propositions over any store, routing to a native `GraphQueryCapable` backend when available. `RetrievalRouter` is the single multi-modal entry point: it checks whether the backing store supports the requested mode (VECTOR / ENTITY / GRAPH_WALK / TEMPORAL / HYBRID) and returns an -empty `supported=false` result rather than falling back to a scan when the mode isn't available. +empty `supported=false` result when the mode isn't available (never falling back to a scan). ```mermaid sequenceDiagram From 18142c21c8a683e375e7a850d980d905d3880ac7 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Wed, 2 Sep 2026 05:39:12 -0400 Subject: [PATCH 06/10] Derive graph ownership from the registered schemas The bookkeeping exclusion was a hand list of four schema objects, and the whole-graph observation read db.labels(), which Neo4j populates for a label the moment a constraint names it, nodes or none. Any store a later slice adds arrived as permanent whole-graph drift: its labels were in the graph and the list had never heard of them. Ownership now derives from the registered DiceStorageSchema beans, so DDL and exclusion come off one list and a schema that registers is a schema the observation excludes, and the observation counts only labels carrying at least one node, because constraint DDL is schema machinery while observation reports data. The guard tests compare the classpath's schema objects against the live context registration, so an unregistered schema fails the build here in place of drifting in production. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- CHANGELOG.md | 53 +++++ .../CollectorAutoConfiguration.kt | 13 +- .../DiceStorageAutoConfiguration.kt | 27 ++- .../DiceStorageSchemaRegistrationTest.kt | 136 +++++++++++++ .../dice/storage/CollectorTraceSchema.kt | 17 +- .../embabel/dice/storage/DiceOwnedSchema.kt | 192 +++++++++++------- .../embabel/dice/storage/DiceStorageSchema.kt | 61 ++++++ .../storage/DrivineObservedSchemaSource.kt | 177 ++++++++-------- .../com/embabel/dice/storage/LineageSchema.kt | 30 ++- .../embabel/dice/storage/MetamodelSchema.kt | 13 +- .../dice/storage/DiceOwnedSchemaTest.kt | 79 +++++-- .../DiceStorageSchemaRegistrationTest.kt | 130 ++++++++++++ .../DrivineDriftCheckIntegrationTest.kt | 12 +- ...vineObservedSchemaSourceIntegrationTest.kt | 109 ++++++++-- .../embabel/dice/storage/TestApplication.kt | 58 +++++- 15 files changed, 876 insertions(+), 231 deletions(-) create mode 100644 dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt create mode 100644 dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt create mode 100644 dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceStorageSchemaRegistrationTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 35abae40..1b6e61b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -444,6 +444,59 @@ and the consumer PRs that deliver it). check's answer is unchanged. Hosts declaring the lineage constraints by hand can swap in `LineageSchema.specs()`. +- The bookkeeping exclusion in the Drivine drift slice above is now derived from the storage + schemas an application registers, and a whole-graph observation counts only labels that carry at + least one node. **EXPERIMENTAL** (shape may change before 1.0): the whole exclusion surface — + `DiceStorageSchema`, `diceStorageCatalog`, and `DiceOwnedSchema`'s instance form — is opt-in + governance wiring that only a host running drift checks touches. + The exclusion used to come off a hand-enumerated list of three schema objects named in + `DiceOwnedSchema`, with a KDoc claiming a new node fragment was the one case needing a line. That + claim held for the three objects named and failed for the fourth object anyone added: a dice store + arriving in another slice got its labels reported as domain drift on every unscoped check, forever, + and both guard tests were built from the same list, so neither could see it. `DiceStorageSchema` is + the contract each store's schema object now implements (`MetamodelSchema`, `CollectorTraceSchema`, + `LineageSchema`), carrying its specs and the relationship types it writes for itself. + `DiceOwnedSchema.of(registered)` reads the beans an application registered, and + `DrivineObservedSchemaSource` takes the result as a required constructor argument, so a store + landing in a later slice takes part by being registered and needs no edit to the drift machinery. + `diceStorageCatalog` builds the Drivine catalog off that same bean list, which is what keeps a + store's constraints and its exclusion from coming apart: one registration produces both. + The KDoc now states the invariant the design can keep — the exclusion covers every schema the + application registered, and a store whose schema is registered nowhere stays visible to + observation, which is the right answer for nodes the application never declared. The four + `@NodeFragment` classes of the core proposition store, and their `HAS_MENTION`/`DERIVED_FROM` + edges, are owned unconditionally, since the observation reads propositions and mentions through + their shapes to answer at all. `INFRASTRUCTURE_LABELS` stays an enumerated list, because a + library's own bookkeeping has no dice schema to derive from. + Second, the label side of a whole-graph observation now keeps only labels some node wears. + `db.labels()` is a catalogue of label *tokens*, and on the `neo4j:2026.05` image these run against + a uniqueness constraint mints its label there on an empty graph, probed directly. So a host + declaring constraints for a type it has not populated reported that type as drift on its first + check after first boot, having stored nothing. Constraint DDL is schema machinery an application + declared; an observation reports what data the graph holds. The check is one label lookup per + label, each stopping at the first node it finds. + Third, both blind guards are replaced by `DiceStorageSchemaRegistrationTest`, in `dice-storage` and + again in `dice-storage-autoconfigure`. It compares two independent things — every + `DiceStorageSchema` singleton a classpath scan finds, and the beans the running Spring context + registered — so a schema object that exists and is wired nowhere fails the build in the slice that + adds it, and a hand-written `SchemaCatalog.of(SomeSchema.specs())` that ensures a dice store's DDL + while leaving it out of the exclusion fails too. A matching scan holds + `DiceOwnedSchema.CORE_NODE_FRAGMENTS` to every `@NodeFragment` in the storage model package. Four + integration cases discriminate the two rules apart: a registered store's constraint-only label is + not observed, its own nodes are excluded once they exist, a label no registered schema declares + still drifts once nodes wear it, and a constraint-minted label nothing wears is not observed even + though dice owns none of it. + **Compatibility: behavioral.** `DrivineObservedSchemaSource` gains a required second constructor + parameter; the top-level `DICE_BOOKKEEPING_RELATIONSHIP_TYPES` is gone, folded into + `DiceOwnedSchema.bookkeepingRelationshipTypes`, and `DiceOwnedSchema` is a class with `of` where it + was an object with `NODE_SHAPES`/`LABELS`. All three arrived in this same Unreleased block, so no + published build carries them. A host wiring the observer registers its dice storage schemas as + `DiceStorageSchema` beans and passes `DiceOwnedSchema.of(schemas)`; `TestApplication` shows the + shape. A whole-graph check reports less than it did in one specific way — labels no node wears + stop appearing — and reports no less about data the graph actually holds. `LineageSchema.specs()` + gained the three lineage range indexes `DiceStorageAutoConfiguration` used to declare separately, + so the DDL a graph-backed host ensures is unchanged and the two lists can no longer disagree. + - `dice-storage-autoconfigure` now depends on `spring-boot-transaction`. On Spring Boot 4, a `PlatformTransactionManager` bean alone does not activate `@Transactional`: the interceptor that reads the annotation lives in that separate module, which was missing here. Every `@Transactional` diff --git a/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/CollectorAutoConfiguration.kt b/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/CollectorAutoConfiguration.kt index 3f415318..27636ff1 100644 --- a/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/CollectorAutoConfiguration.kt +++ b/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/CollectorAutoConfiguration.kt @@ -33,7 +33,9 @@ import com.embabel.dice.spi.ConnectedComponentsFinder import com.embabel.dice.spi.InMemoryCollectorTraceStore import com.embabel.dice.spi.InMemoryConnectedComponentsFinder import com.embabel.dice.storage.CollectorTraceSchema +import com.embabel.dice.storage.DiceStorageSchema import com.embabel.dice.storage.DrivineCollectorTraceStore +import com.embabel.dice.storage.diceStorageCatalog import org.drivine.manager.PersistenceManager import org.drivine.schema.SchemaCatalog import org.slf4j.LoggerFactory @@ -101,9 +103,18 @@ class CollectorAutoConfiguration( return InMemoryCollectorTraceStore() } + /** + * The trace store's schema, registered as a [DiceStorageSchema] so one bean answers both + * questions about it: Drivine ensures its constraints and indexes, and dice's drift observation + * reads the same object to recognise the trace store's nodes and edges as its own. + */ @Bean @ConditionalOnProperty(prefix = "embabel.dice.store", name = ["type"], havingValue = "graph") - fun collectorTraceSchema(): SchemaCatalog = SchemaCatalog.of(*CollectorTraceSchema.specs().toTypedArray()) + fun collectorTraceStorageSchema(): DiceStorageSchema = CollectorTraceSchema + + @Bean + @ConditionalOnProperty(prefix = "embabel.dice.store", name = ["type"], havingValue = "graph") + fun collectorTraceSchema(): SchemaCatalog = diceStorageCatalog(listOf(CollectorTraceSchema)) // ---- Built-in pair source ---- diff --git a/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageAutoConfiguration.kt b/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageAutoConfiguration.kt index e2fac013..9f34969e 100644 --- a/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageAutoConfiguration.kt +++ b/dice-storage-autoconfigure/src/main/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageAutoConfiguration.kt @@ -29,11 +29,14 @@ import com.embabel.dice.proposition.DecaySweepConfig import com.embabel.dice.proposition.PropositionRepository import com.embabel.dice.proposition.store.InMemoryDecayManager import com.embabel.dice.proposition.store.InMemoryPropositionRepository +import com.embabel.dice.storage.DiceStorageSchema import com.embabel.dice.storage.DrivineChunkHistoryStore import com.embabel.dice.storage.DrivineCollectorRecordStore import com.embabel.dice.storage.DrivinePropositionRepository import com.embabel.dice.storage.DrivineProjectionRecordStore import com.embabel.dice.storage.GraphDecayManager +import com.embabel.dice.storage.LineageSchema +import com.embabel.dice.storage.diceStorageCatalog import org.drivine.manager.GraphObjectManager import org.drivine.manager.PersistenceManager import org.drivine.schema.RangeIndexSpec @@ -139,17 +142,23 @@ class DiceStorageAutoConfiguration { persistenceManager: PersistenceManager, ): CollectorRecordStore = DrivineCollectorRecordStore(persistenceManager) + /** + * The lineage stores' schema, registered as a [DiceStorageSchema] so one bean answers both + * questions about it: Drivine ensures its constraints and indexes, and dice's drift observation + * reads the same object to recognise `(:ProjectionRecord)` and `(:CollectorRecord)` nodes as its + * own. Registering the DDL by hand here, as this used to, left the second half to a separate + * list that could fall behind. + * + * Natural keys back the MERGE upserts, so a replayed record updates in place and never + * duplicates; the range indexes back the lineage lookups. Both live in [LineageSchema]. + */ @Bean @ConditionalOnProperty(prefix = "embabel.dice.store", name = ["type"], havingValue = "graph") - fun lineageRecordSchema(): SchemaCatalog = SchemaCatalog.of( - // Natural keys back the MERGE upserts: a replayed record updates in place, not duplicates. - UniquenessConstraintSpec(label = "ProjectionRecord", properties = listOf("propositionId", "runId", "target")), - UniquenessConstraintSpec(label = "CollectorRecord", properties = listOf("propositionId", "runId")), - UniquenessConstraintSpec(label = "CollectorRun", property = "runId"), - RangeIndexSpec("ProjectionRecord", "propositionId"), - RangeIndexSpec("ProjectionRecord", "lifecycle"), - RangeIndexSpec("CollectorRecord", "propositionId"), - ) + fun lineageStorageSchema(): DiceStorageSchema = LineageSchema + + @Bean + @ConditionalOnProperty(prefix = "embabel.dice.store", name = ["type"], havingValue = "graph") + fun lineageRecordSchema(): SchemaCatalog = diceStorageCatalog(listOf(LineageSchema)) @Bean @ConditionalOnBean(Ai::class) diff --git a/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt b/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt new file mode 100644 index 00000000..336fc2d3 --- /dev/null +++ b/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt @@ -0,0 +1,136 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage.autoconfigure + +import com.embabel.dice.storage.CollectorTraceSchema +import com.embabel.dice.storage.DiceStorageSchema +import com.embabel.dice.storage.LineageSchema +import org.assertj.core.api.Assertions.assertThat +import org.drivine.manager.GraphObjectManager +import org.drivine.manager.PersistenceManager +import org.drivine.schema.SchemaCatalog +import org.drivine.schema.SchemaItemSpec +import org.junit.jupiter.api.Test +import org.mockito.kotlin.mock +import org.springframework.beans.factory.getBeanProvider +import org.springframework.boot.autoconfigure.AutoConfigurations +import org.springframework.boot.test.context.runner.ApplicationContextRunner +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider +import org.springframework.context.annotation.Configuration +import org.springframework.core.type.filter.AssignableTypeFilter + +/** + * The autoconfigure half of the registration guard `dice-storage` states in full. + * + * A whole-graph drift check hides dice's own storage by reading the [DiceStorageSchema] beans the + * application registered, so a store whose schema reaches the database through a hand-written + * catalog gets its constraints — which is what mints its labels in Neo4j — while staying outside the + * exclusion. Its nodes are then reported as domain drift on every check. That is the shape of the + * defect this whole change closes, and this test is what makes it fail loudly here. + * + * The rule checked is one-directional on purpose. This module wires the proposition store, the + * lineage stores and the collector trace store; the metamodel governance stores have no wiring here + * yet, so requiring every dice schema on the classpath to be registered would demand DDL for stores + * this module never creates. What it does require is that nothing this module *does* ensure against + * a database belongs to a dice store it left out of the exclusion. + */ +class DiceStorageSchemaRegistrationTest { + + private val graphRunner = ApplicationContextRunner() + .withConfiguration( + AutoConfigurations.of( + DiceStorageAutoConfiguration::class.java, + CollectorAutoConfiguration::class.java, + ), + ) + .withUserConfiguration(StubDrivineConfig::class.java) + .withPropertyValues("embabel.dice.store.type=graph") + + @Test + fun `no dice store's schema reaches the database without contributing to ownership`() { + graphRunner.run { ctx -> + val registered = ctx.getBeanProvider().orderedStream().toList() + .map { it::class } + .toSet() + val ensured: Set = ctx.getBeanProvider().orderedStream().toList() + .flatMap { catalog -> catalog.items } + .toSet() + + diceStorageSchemasOnClasspath() + .filter { schema -> schema::class !in registered } + .forEach { unregistered -> + assertThat(unregistered.specs().filter { spec -> spec in ensured }) + .describedAs( + "%s is ensured against the database without being registered as a " + + "DiceStorageSchema, so every label it declares drifts", + unregistered::class.simpleName, + ) + .isEmpty() + } + } + } + + @Test + fun `the graph backend registers the schemas of the stores it wires`() { + // Named explicitly, because the rule above passes trivially for a module that registers + // nothing. These are the two dice stores this module's graph backend creates beans for. + graphRunner.run { ctx -> + val registered = ctx.getBeanProvider().orderedStream().toList() + + assertThat(registered).contains(LineageSchema, CollectorTraceSchema) + } + } + + @Test + fun `the in-memory backend registers no graph schema at all`() { + ApplicationContextRunner() + .withConfiguration( + AutoConfigurations.of( + DiceStorageAutoConfiguration::class.java, + CollectorAutoConfiguration::class.java, + ), + ) + .run { ctx -> + assertThat(ctx.getBeanProvider().orderedStream().toList()).isEmpty() + assertThat(ctx.getBeanProvider().orderedStream().toList()).isEmpty() + } + } + + /** + * The Drivine handles the graph stores are wired with. Nothing here talks to a database: the + * question is which schema beans the autoconfiguration registers, and the stores only have to be + * constructible for their conditions to resolve. + */ + @Configuration + open class StubDrivineConfig { + + @Bean + open fun persistenceManager(): PersistenceManager = mock() + + @Bean + open fun graphObjectManager(): GraphObjectManager = mock() + } + + /** Every `DiceStorageSchema` singleton on this module's classpath. */ + private fun diceStorageSchemasOnClasspath(): List { + val scanner = ClassPathScanningCandidateComponentProvider(false) + scanner.addIncludeFilter(AssignableTypeFilter(DiceStorageSchema::class.java)) + return scanner.findCandidateComponents("com.embabel.dice.storage") + .mapNotNull { definition -> definition.beanClassName } + .mapNotNull { name -> Class.forName(name).kotlin.objectInstance as DiceStorageSchema? } + } +} diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceSchema.kt index 8883f96c..6f520222 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/CollectorTraceSchema.kt @@ -20,13 +20,20 @@ import org.drivine.schema.SchemaItemSpec import org.drivine.schema.UniquenessConstraintSpec /** - * The constraints and indexes [DrivineCollectorTraceStore] needs, factored out as plain data so - * the integration-test harness and the autoconfigure module can share one source of truth - * instead of redeclaring the schema twice. + * The constraints, indexes and edge types [DrivineCollectorTraceStore] needs, as plain data the + * integration-test harness and the autoconfigure module both register, so the schema is declared + * once for everyone who ensures it. */ -object CollectorTraceSchema { +object CollectorTraceSchema : DiceStorageSchema { - fun specs(): List = listOf( + /** + * The edges the trace store writes between its own nodes: a signal score points at the + * candidate edge it scored, and a retired proposition at the decision that retired it. Both sit + * entirely inside the trace store's own nodes, so a whole-graph observation hides them. + */ + override val bookkeepingRelationshipTypes: Set = setOf("SCORED", "RETIRED_IN") + + override fun specs(): List = listOf( // Natural-key uniqueness, one per node label. UniquenessConstraintSpec(label = "CollectorTraceRun", property = "runId"), UniquenessConstraintSpec(label = "CollectorCandidateEdge", property = "id"), diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt index 298578d7..119f3814 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt @@ -31,7 +31,8 @@ import kotlin.reflect.full.primaryConstructor /** * What a node in a graph has to look like for dice to own it: the labels dice's own storage writes, - * and, per label, the properties dice writes on every such node. + * and, per label, the properties dice writes on every such node. Plus the relationship types dice + * writes for its own bookkeeping. * * ## Why ownership needs saying at all * @@ -39,14 +40,27 @@ import kotlin.reflect.full.primaryConstructor * shares that graph with dice's own storage. Nothing dice writes for itself belongs to a declared * domain schema, so counting those nodes would report dice's bookkeeping as drift on every run, and * would make governance report itself: stamping a version and writing a drift report both add nodes - * to the graph the next check observes. `DrivineObservedSchemaSource` subtracts what this object - * describes. + * to the graph the next check observes. `DrivineObservedSchemaSource` subtracts what an instance of + * this class describes. * * Names alone can't decide it. An app is free to govern its own type called `Source`, and hiding * every `(:Source)` node would hide that type from every report it should appear in. So ownership is * decided by shape: a label counts as dice's only while every node wearing it carries the properties * dice's own writer always writes. * + * ## The real invariant: ownership covers what the application registered + * + * [of] takes the [DiceStorageSchema] beans the running application registered, so the exclusion + * covers every dice store that application wired, whenever it was written. A store added by a later + * slice takes part by being registered, with no edit here. + * + * The converse is the honest half, and it is the behaviour to expect: **a dice store whose schema + * the application never registered stays visible to observation.** Its nodes are then reported as + * domain drift, which is right — an application that never declared the store to Drivine has told + * dice nothing about those nodes, and an observation reporting an unexplained label is the correct + * answer. The way to hide a store's nodes is to register its schema, which is the same act that + * gives the store its constraints; see [diceStorageCatalog]. + * * ## The shapes come from the schema definitions * * Nothing here is a hand-kept list of property names. Each shape is read out of the definition the @@ -58,13 +72,16 @@ import kotlin.reflect.full.primaryConstructor * back on. Optional and nullable ones stay out, because a node dice wrote is allowed to be missing * them, and demanding one would make dice's own nodes look foreign and bring back the * self-reporting case above. - * - **Cypher-backed stores** ([MetamodelSchema], [CollectorTraceSchema], [LineageSchema]) declare - * their labels and natural keys as uniqueness constraints. The shape is the union of the key - * properties for that label, which is exactly what those stores MERGE on, so every node they - * create carries all of them. + * - **Cypher-backed stores** — every registered [DiceStorageSchema] — declare their labels and + * natural keys as uniqueness constraints. The shape is the union of the key properties for that + * label, which is exactly what those stores MERGE on, so every node they create carries all of + * them. * - * Adding a node label to any of those definitions carries its shape here with it. A new node - * fragment is the one case needing a line: add its class to [NODE_FRAGMENTS]. + * The four node fragments are in every ownership set, registered or otherwise. They are the core + * proposition store, and `DrivineObservedSchemaSource` reads propositions and mentions through their + * shapes to answer what a graph's extractions claimed, so an observation that could not recognise + * them could not run at all. `DiceStorageSchemaRegistrationTest` fails when a new `@NodeFragment` + * class appears in the storage model package and is missing from [CORE_NODE_FRAGMENTS]. * * ## Where the boundary genuinely blurs * @@ -76,94 +93,113 @@ import kotlin.reflect.full.primaryConstructor * domain `(:Source {key})` with no `kind` keeps `Source` observable, as does any other domain node * missing a property dice always writes. Context-scoped observation avoids the question entirely, * since it reads mention types and edges marked as projected from domain data. + * + * @property nodeShapes Every node label dice writes, with the properties that identify a node + * carrying that label as dice's own. A node missing any of them was written by somebody else. + * @property bookkeepingRelationshipTypes Every relationship type dice writes for itself. */ -object DiceOwnedSchema { - - /** - * The Drivine node fragments dice persists directly. Each one names its own label and carries - * its own properties, so this list holds classes and no strings. - */ - private val NODE_FRAGMENTS: List> = listOf( - PropositionNode::class, - Mention::class, - SourceNode::class, - ProcessedChunkNode::class, - ) - - /** - * Every node label dice writes, with the properties that identify a node carrying that label as - * dice's own. A node missing any of them was written by somebody else. - */ - val NODE_SHAPES: Map> = buildMap { - NODE_FRAGMENTS.forEach { fragment -> putAll(shapesOf(fragment)) } - putAll(keyShapesOf(MetamodelSchema.specs())) - putAll(keyShapesOf(CollectorTraceSchema.specs())) - putAll(keyShapesOf(LineageSchema.specs())) - } +class DiceOwnedSchema private constructor( + val nodeShapes: Map>, + val bookkeepingRelationshipTypes: Set, +) { /** The label names, for callers that only need the names. */ - val LABELS: Set = NODE_SHAPES.keys + val labels: Set = nodeShapes.keys /** * A Cypher predicate that holds for a node dice owns: every property of the label's shape is * present. * - * Label and property names are compile-time constants of the definitions above, so nothing + * Label and property names come from the schema definitions dice compiled, so nothing * caller-derived is assembled into Cypher. * * @param alias The variable the node is bound to. - * @param label One of [LABELS]. + * @param label One of [labels]. * @return The predicate text, ready to follow a `WHERE`. */ fun ownedNodePredicate(alias: String, label: String): String { - val shape = requireNotNull(NODE_SHAPES[label]) { "'$label' is no dice storage label" } + val shape = requireNotNull(nodeShapes[label]) { "'$label' is no dice storage label" } return shape.joinToString(" AND ") { property -> "$alias.$property IS NOT NULL" } } - /** The label a fragment writes, mapped to the properties dice always writes on it. */ - private fun shapesOf(fragment: KClass<*>): Map> { - val labels = fragment.annotations.filterIsInstance().singleOrNull()?.labels - ?: error("${fragment.simpleName} carries no @NodeFragment") - val shape = alwaysWritten(fragment) - return labels.associateWith { shape } - } + companion object { - /** - * The fragment's properties dice's writer cannot leave out: declared non-null, with no default - * value, and held in the node as a single property. - * - * A default value is what tells us a property is optional. Drivine's `@Default` and - * `@EmptyWhenAbsent` both go on parameters that carry one, so this covers them, along with - * `@PropertyBag`, whose map is spread across `metadata.` properties and appears under no - * name of its own. - */ - private fun alwaysWritten(fragment: KClass<*>): List { - val constructor = fragment.primaryConstructor - ?: error("${fragment.simpleName} has no primary constructor") - return constructor.parameters - .filter { parameter -> !parameter.isOptional && !parameter.type.isMarkedNullable } - .filter { parameter -> isSingleProperty(parameter.type) } - .mapNotNull { parameter -> parameter.name } - } + /** + * The Drivine node fragments dice persists directly. Each one names its own label and + * carries its own properties, so this list holds classes and no strings. + */ + val CORE_NODE_FRAGMENTS: List> = listOf( + PropositionNode::class, + Mention::class, + SourceNode::class, + ProcessedChunkNode::class, + ) - /** - * Whether a value of this type lands in the node as one property. Collections, maps and nested - * fragments do something else with it, so they take no part in a shape. - */ - private fun isSingleProperty(type: KType): Boolean { - val classifier = type.classifier as? KClass<*> ?: return false - if (classifier.isSubclassOf(Collection::class) || classifier.isSubclassOf(Map::class)) return false - return !classifier.hasAnnotation() && !classifier.hasAnnotation() + /** + * The edges the core proposition store writes: a proposition to each of its mentions, and a + * proposition to the source it came from. Every graph dice has ever written to holds them. + */ + val CORE_RELATIONSHIP_TYPES: Set = setOf("HAS_MENTION", "DERIVED_FROM") + + /** + * Ownership for an application that registered [schemas]. + * + * @param schemas The [DiceStorageSchema] beans the application registered, in any order. + */ + fun of(schemas: List): DiceOwnedSchema = DiceOwnedSchema( + nodeShapes = buildMap { + CORE_NODE_FRAGMENTS.forEach { fragment -> putAll(shapesOf(fragment)) } + schemas.forEach { schema -> putAll(keyShapesOf(schema.specs())) } + }, + bookkeepingRelationshipTypes = CORE_RELATIONSHIP_TYPES + + schemas.flatMap { schema -> schema.bookkeepingRelationshipTypes }, + ) } +} - /** - * The labels a Cypher-backed store declares, each mapped to every property its uniqueness - * constraints name. Those properties are the store's MERGE keys, so a node it created carries - * all of them. Range indexes are left alone, since they cover properties a record can be - * missing. - */ - private fun keyShapesOf(specs: List): Map> = - specs.filterIsInstance() - .groupBy { spec -> spec.label } - .mapValues { (_, group) -> group.flatMap { spec -> spec.properties }.distinct() } +/** The label a fragment writes, mapped to the properties dice always writes on it. */ +private fun shapesOf(fragment: KClass<*>): Map> { + val labels = fragment.annotations.filterIsInstance().singleOrNull()?.labels + ?: error("${fragment.simpleName} carries no @NodeFragment") + val shape = alwaysWritten(fragment) + return labels.associateWith { shape } } + +/** + * The fragment's properties dice's writer cannot leave out: declared non-null, with no default + * value, and held in the node as a single property. + * + * A default value is what tells us a property is optional. Drivine's `@Default` and + * `@EmptyWhenAbsent` both go on parameters that carry one, so this covers them, along with + * `@PropertyBag`, whose map is spread across `metadata.` properties and appears under no + * name of its own. + */ +private fun alwaysWritten(fragment: KClass<*>): List { + val constructor = fragment.primaryConstructor + ?: error("${fragment.simpleName} has no primary constructor") + return constructor.parameters + .filter { parameter -> !parameter.isOptional && !parameter.type.isMarkedNullable } + .filter { parameter -> isSingleProperty(parameter.type) } + .mapNotNull { parameter -> parameter.name } +} + +/** + * Whether a value of this type lands in the node as one property. Collections, maps and nested + * fragments do something else with it, so they take no part in a shape. + */ +private fun isSingleProperty(type: KType): Boolean { + val classifier = type.classifier as? KClass<*> ?: return false + if (classifier.isSubclassOf(Collection::class) || classifier.isSubclassOf(Map::class)) return false + return !classifier.hasAnnotation() && !classifier.hasAnnotation() +} + +/** + * The labels a Cypher-backed store declares, each mapped to every property its uniqueness + * constraints name. Those properties are the store's MERGE keys, so a node it created carries + * all of them. Range indexes are left alone, since they cover properties a record can be + * missing. + */ +private fun keyShapesOf(specs: List): Map> = + specs.filterIsInstance() + .groupBy { spec -> spec.label } + .mapValues { (_, group) -> group.flatMap { spec -> spec.properties }.distinct() } diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt new file mode 100644 index 00000000..9fef5361 --- /dev/null +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt @@ -0,0 +1,61 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import org.drivine.schema.SchemaCatalog +import org.drivine.schema.SchemaItemSpec + +/** + * One dice-owned Cypher-backed store's schema: the constraints and indexes it needs, and the + * relationship types it writes for dice's own bookkeeping. + * + * An application registers these as beans, and the registered set does two jobs at once. Drivine + * ensures the DDL from it ([diceStorageCatalog]), and [DiceOwnedSchema] reads the same set to work + * out which nodes and edges in a shared graph are dice's own, so a whole-graph drift check leaves + * them alone. Making one list answer both questions is the point: a store whose schema reaches the + * database has, by construction, reached the ownership rule too. + * + * Adding a store is therefore a two-file job in the slice that adds it: declare its schema object + * beside the store, and register it where the application wires its stores. Nothing in the drift + * machinery has to hear about it. + */ +interface DiceStorageSchema { + + /** The constraints and indexes this store's writes depend on. */ + fun specs(): List + + /** + * Relationship types this store writes between its own nodes. A whole-graph observation hides + * them, on the same grounds it hides the node labels: they are dice's bookkeeping, and no + * declared domain schema is expected to mention them. + * + * Empty for a store that writes no edges of its own. + */ + val bookkeepingRelationshipTypes: Set + get() = emptySet() +} + +/** + * The Drivine catalog for a set of registered dice storage schemas. + * + * Wiring the DDL through this function is what keeps a store's constraints and its ownership + * exclusion off one list. A host that hand-writes `SchemaCatalog.of(SomeSchema.specs())` gets the + * constraints while leaving the store out of the exclusion, and its labels then read as domain + * drift for as long as the graph holds its nodes; `DiceStorageSchemaRegistrationTest` fails when + * that happens. + */ +fun diceStorageCatalog(schemas: List): SchemaCatalog = + SchemaCatalog.of(schemas.flatMap { schema -> schema.specs() }) diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt index 4322a442..5ab44771 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt @@ -23,26 +23,6 @@ import org.drivine.query.QuerySpecification import org.springframework.transaction.annotation.Transactional import java.time.Clock -/** - * Relationship types dice writes for its own bookkeeping, on the same grounds as - * [DiceOwnedSchema] and under the same ownership rule. - * - * Dice's bookkeeping extends past node labels. `HAS_MENTION` and `DERIVED_FROM` sit on every - * proposition ever stored, so without this set a whole-graph observation reports them as undeclared - * relationship drift on every run against a populated graph. - * - * These carry no per-type shape because one discriminator covers all of them, and it lives on the - * edge: every edge the graph writer projects from domain data carries `sourcePropositions`, and no - * dice bookkeeping edge carries it. The context-scoped path selects on the same marker, so both - * paths agree. - */ -val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( - "HAS_MENTION", - "DERIVED_FROM", - "SCORED", - "RETIRED_IN", -) - /** * Node labels written by infrastructure libraries that Dice runs on top of, excluding them from * whole-graph drift observation. @@ -50,10 +30,12 @@ val DICE_BOOKKEEPING_RELATIONSHIP_TYPES: Set = setOf( * Infrastructure libraries own these labels and write them as part of their own bookkeeping. * DICE's schema definitions cannot declare them — the library owns them entirely — so every * whole-graph observation would report them as undeclared drift on every run if left unexcluded. + * This one stays a hand-kept list for exactly that reason: everything else dice excludes is derived + * from a schema dice compiled, and a library's own bookkeeping has no such schema to derive from. * - * Unlike [DICE_BOOKKEEPING_RELATIONSHIP_TYPES], which are decided by shape (a `sourcePropositions` - * marker on edges), these are decided by name. An infrastructure library's bookkeeping is not - * declared anywhere in DICE's schema, so there is no shape to recognize it by. + * Dice's own bookkeeping edges are decided by shape (a `sourcePropositions` marker on edges), while + * these are decided by name. An infrastructure library's bookkeeping is declared nowhere in DICE's + * schema, so there is no shape to recognize it by. * * Currently includes `_DrivineSchema`, written by the Drivine library itself. */ @@ -68,9 +50,9 @@ val INFRASTRUCTURE_LABELS: Set = setOf( * There are two observation paths, because the database offers no single query that answers both: * * - **Whole graph** (`contextId == null`) reads the database's own catalogue, `db.labels()` and - * `db.relationshipTypes()`, and subtracts what dice owns from both sides. Ownership goes by node - * shape; see [DiceOwnedSchema]. A dice label the domain is also using stays in the observation, so - * an undeclared type can still be reported. + * `db.relationshipTypes()`, keeps the labels that carry at least one node, and subtracts what dice + * owns from both sides. Ownership goes by node shape; see [DiceOwnedSchema]. A dice label the + * domain is also using stays in the observation, so an undeclared type can still be reported. * * It then asks a second question, and reports the answer in its own set: the distinct * `Mention.type` values on dice's own propositions, across the whole graph, returned as @@ -105,6 +87,17 @@ val INFRASTRUCTURE_LABELS: Set = setOf( * `Person` with parent label `Agent` and declares no `Agent` type. An unscoped check and a * context-scoped one now read mention types the same strict way. * + * ## A label with no nodes is no observation + * + * The whole-graph label side counts only labels carrying at least one node. Neo4j's `db.labels()` + * is a catalogue of label *tokens*, and a token is minted the moment a constraint or an index names + * a label, before any node wears it — probed directly against the `neo4j:2026.05` image this runs + * on, where a uniqueness constraint on an empty label puts that label in `db.labels()` for good. + * Constraint DDL is schema machinery an application declared; an observation reports what data the + * graph holds. So a type a host has declared constraints for and never populated is silently clean, + * which is the honest answer: there is nothing there to have drifted. The check costs one label + * lookup per label, each stopping at the first node it finds. + * * Two limits follow from working off names and shape: * * 1. Ownership is decided per label, and never per node. If any node wearing a dice label fails @@ -115,72 +108,92 @@ val INFRASTRUCTURE_LABELS: Set = setOf( * don't pay it. * * @param persistenceManager Drivine's handle on the `neo` datasource. + * @param ownedSchema What this application's own dice storage looks like, built from the + * [DiceStorageSchema] beans it registered. Required, with no default, because an observer that + * guessed at ownership would report some other slice's store as domain drift forever; whoever + * wires an observer has to say what dice owns in that application. * @param clock Supplies the snapshot's capture instant. Injectable because that instant ends up in * a drift report's natural key, so a test has to be able to choose whether two checks record one * observation or two. */ open class DrivineObservedSchemaSource( private val persistenceManager: PersistenceManager, + private val ownedSchema: DiceOwnedSchema, private val clock: Clock = Clock.systemUTC(), ) : ObservedSchemaSource { - private companion object { + /** + * Every label some node in the graph actually wears. + * + * `db.labels()` alone answers with schema tokens, including labels a constraint or index minted + * on an empty graph; see the class doc. The subquery turns each token into a question about + * data, stopping at the first node wearing it, so the answer describes what the graph holds. + */ + private val labelsWithNodes = """ + CALL db.labels() YIELD label + CALL (label) { + MATCH (n:${'$'}(label)) + RETURN n + LIMIT 1 + } + RETURN DISTINCT label + """.trimIndent() - private const val ALL_LABELS = "CALL db.labels() YIELD label RETURN label" + /** + * Dice labels the domain has also claimed: those carrying at least one node that fails + * dice's shape for them. Whatever this returns stays in the observation. + * + * One branch per label, each stopping at the first non-conforming node, unioned into a + * single round trip. Label and property names come from [DiceOwnedSchema], which derives + * them from the registered storage definitions; nothing caller-derived is assembled in. + */ + private val labelsClaimedByDomain: String = ownedSchema.nodeShapes.entries + .joinToString("\nUNION ALL\n") { (label, shape) -> + val notDiceShaped = shape.joinToString(" OR ") { property -> "n.$property IS NULL" } + "MATCH (n:$label) WHERE $notDiceShaped RETURN '$label' AS label LIMIT 1" + } - private const val ALL_RELATIONSHIP_TYPES = - "CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType" + /** + * Bookkeeping relationship types the domain has also claimed: those carrying at least one + * edge with `sourcePropositions`, the property the graph writer stamps on every edge it + * projects from domain data. + */ + private val relationshipTypesClaimedByDomain: String = """ + MATCH ()-[r:${ownedSchema.bookkeepingRelationshipTypes.joinToString("|")}]->() + WHERE r.sourcePropositions IS NOT NULL + RETURN DISTINCT type(r) + """.trimIndent() - /** - * Dice labels the domain has also claimed: those carrying at least one node that fails - * dice's shape for them. Whatever this returns stays in the observation. - * - * One branch per label, each stopping at the first non-conforming node, unioned into a - * single round trip. Label and property names come from [DiceOwnedSchema], which derives - * them from the storage definitions; nothing caller-derived is assembled in. - */ - private val LABELS_CLAIMED_BY_DOMAIN: String = DiceOwnedSchema.NODE_SHAPES.entries - .joinToString("\nUNION ALL\n") { (label, shape) -> - val notDiceShaped = shape.joinToString(" OR ") { property -> "n.$property IS NULL" } - "MATCH (n:$label) WHERE $notDiceShaped RETURN '$label' AS label LIMIT 1" - } + /** + * Entity types across the whole graph: what dice's own propositions mention, wherever they + * live. + * + * The catalogue this file reads for labels knows nothing about mention types. A type an + * extractor wrote reaches `db.labels()` only if something projected a node for it, so a + * graph can hold active propositions mentioning `Ghost` with no `(:Ghost)` node anywhere, + * and a whole-graph check reading labels alone calls that graph clean. This query asks the + * propositions themselves. + * + * Both ends are held to dice's own shape, so a domain node that happens to wear + * `:Proposition` or `:Mention` contributes nothing: what comes back is the set of types + * dice's own extraction recorded. `m.type` is part of the mention shape, so a mention with + * no type is already excluded by it. + * + * Every proposition counts, whatever its status, which is how the context-scoped query + * reads too. A quarantined proposition still carries the undeclared type it was quarantined + * for, and a check that stopped reporting it would read clean while the data sits there. + */ + private val mentionTypesInGraph = """ + MATCH (p:Proposition)-[:HAS_MENTION]->(m:Mention) + WHERE ${ownedSchema.ownedNodePredicate("p", "Proposition")} + AND ${ownedSchema.ownedNodePredicate("m", "Mention")} + RETURN DISTINCT m.type + """.trimIndent() - /** - * Bookkeeping relationship types the domain has also claimed: those carrying at least one - * edge with `sourcePropositions`, the property the graph writer stamps on every edge it - * projects from domain data. - */ - private val RELATIONSHIP_TYPES_CLAIMED_BY_DOMAIN: String = """ - MATCH ()-[r:${DICE_BOOKKEEPING_RELATIONSHIP_TYPES.joinToString("|")}]->() - WHERE r.sourcePropositions IS NOT NULL - RETURN DISTINCT type(r) - """.trimIndent() + private companion object { - /** - * Entity types across the whole graph: what dice's own propositions mention, wherever they - * live. - * - * The catalogue this file reads for labels knows nothing about mention types. A type an - * extractor wrote reaches `db.labels()` only if something projected a node for it, so a - * graph can hold active propositions mentioning `Ghost` with no `(:Ghost)` node anywhere, - * and a whole-graph check reading labels alone calls that graph clean. This query asks the - * propositions themselves. - * - * Both ends are held to dice's own shape, so a domain node that happens to wear - * `:Proposition` or `:Mention` contributes nothing: what comes back is the set of types - * dice's own extraction recorded. `m.type` is part of the mention shape, so a mention with - * no type is already excluded by it. - * - * Every proposition counts, whatever its status, which is how the context-scoped query - * reads too. A quarantined proposition still carries the undeclared type it was quarantined - * for, and a check that stopped reporting it would read clean while the data sits there. - */ - private val MENTION_TYPES_IN_GRAPH = """ - MATCH (p:Proposition)-[:HAS_MENTION]->(m:Mention) - WHERE ${DiceOwnedSchema.ownedNodePredicate("p", "Proposition")} - AND ${DiceOwnedSchema.ownedNodePredicate("m", "Mention")} - RETURN DISTINCT m.type - """.trimIndent() + private const val ALL_RELATIONSHIP_TYPES = + "CALL db.relationshipTypes() YIELD relationshipType RETURN relationshipType" /** * Entity types in one context: what its propositions actually mention. @@ -257,19 +270,19 @@ open class DrivineObservedSchemaSource( private fun observeWholeGraph(): ObservedSchema { // Subtract only the storage the domain has not also claimed. A name both dice and the // domain use counts as the domain's here, so it stays observable as drift. - val hiddenLabels = DiceOwnedSchema.LABELS - queryStrings(LABELS_CLAIMED_BY_DOMAIN) + val hiddenLabels = ownedSchema.labels - queryStrings(labelsClaimedByDomain) val hiddenRelationshipTypes = - DICE_BOOKKEEPING_RELATIONSHIP_TYPES - queryStrings(RELATIONSHIP_TYPES_CLAIMED_BY_DOMAIN) + ownedSchema.bookkeepingRelationshipTypes - queryStrings(relationshipTypesClaimedByDomain) // Two kinds of entity name, in two sets: the labels the graph reports, and the types dice's // propositions were extracted with. The mention side needs no subtraction, since the query // that produced it already asked dice's own propositions, and it stays out of the label set // so the differ can hold it to the mention rule; see the class doc. return ObservedSchema( - entityTypeNames = queryStrings(ALL_LABELS) - hiddenLabels - INFRASTRUCTURE_LABELS, + entityTypeNames = queryStrings(labelsWithNodes) - hiddenLabels - INFRASTRUCTURE_LABELS, relationshipTypeNames = queryStrings(ALL_RELATIONSHIP_TYPES) - hiddenRelationshipTypes, capturedAt = clock.instant(), entityTypeBasis = ObservedSchema.EntityTypeBasis.GRAPH_LABELS, - mentionTypeNames = queryStrings(MENTION_TYPES_IN_GRAPH), + mentionTypeNames = queryStrings(mentionTypesInGraph), ) } diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt index 6cf88714..6914a41f 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt @@ -15,6 +15,7 @@ */ package com.embabel.dice.storage +import org.drivine.schema.RangeIndexSpec import org.drivine.schema.SchemaItemSpec import org.drivine.schema.UniquenessConstraintSpec @@ -24,17 +25,17 @@ import org.drivine.schema.UniquenessConstraintSpec * * Three things read this and have to agree. [DrivineProjectionRecordStore] and * [DrivineCollectorRecordStore] build their MERGE patterns from [mergePattern], so a record is - * always upserted on the key named here. A host, and the integration-test harness, declares [specs] - * so those MERGEs are race-free: a MERGE on an unconstrained key lets concurrent writers all miss - * the match, all create, and fill the lineage table with duplicates. [DiceOwnedSchema] reads the - * same keys to work out which `(:ProjectionRecord)` and `(:CollectorRecord)` nodes in a graph are - * dice's own. + * always upserted on the key named here. A host, and the integration-test harness, registers this + * object as a [DiceStorageSchema] bean, which is what makes those MERGEs race-free: a MERGE on an + * unconstrained key lets concurrent writers all miss the match, all create, and fill the lineage + * table with duplicates. The same registration tells [DiceOwnedSchema] which + * `(:ProjectionRecord)` and `(:CollectorRecord)` nodes in a graph are dice's own. * * Keeping all three off one map is what stops them drifting apart. A key that appears in a store's * Cypher and nowhere else can lose its constraint, or stop matching the shape an observation * recognises, with nothing failing to say so. */ -object LineageSchema { +object LineageSchema : DiceStorageSchema { /** One node per projection outcome. */ const val PROJECTION_RECORD: String = "ProjectionRecord" @@ -58,10 +59,21 @@ object LineageSchema { /** Every node label the lineage stores write, for test cleanup and for drift exclusion. */ val LABELS: List = NATURAL_KEYS.keys.toList() - /** A uniqueness constraint per natural key, which is what makes the stores' MERGEs race-free. */ - fun specs(): List = NATURAL_KEYS.map { (label, key) -> + /** + * A uniqueness constraint per natural key, which is what makes the stores' MERGEs race-free, + * plus the range indexes the lineage reads look records up by. + * + * The lookup indexes live here alongside the constraints so one registration covers everything + * these stores need. A host that ensures the constraints while missing the indexes gets correct + * answers off full scans, which is the kind of gap a hand-copied schema list produces. + */ + override fun specs(): List = NATURAL_KEYS.map { (label, key) -> UniquenessConstraintSpec(label = label, properties = key) - } + } + listOf( + RangeIndexSpec(PROJECTION_RECORD, "propositionId"), + RangeIndexSpec(PROJECTION_RECORD, "lifecycle"), + RangeIndexSpec(COLLECTOR_RECORD, "propositionId"), + ) /** * The `(alias:Label {property: $property, ...})` pattern a store MERGEs on. diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt index 91180e70..a6d68d2f 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt @@ -21,13 +21,12 @@ import org.drivine.schema.UniquenessConstraintSpec /** * The constraints and node labels the metamodel governance stores need, as plain data. * - * Two things depend on this list and have to agree. A host, and the integration-test harness, - * declares [specs] so the stores' MERGEs are race-free; [LABELS] is what - * [DrivineObservedSchemaSource] subtracts from an observation, which keeps governance from reporting - * its own bookkeeping as domain drift. Keeping both here makes adding a governance node label a - * single edit in one module. + * A host, and the integration-test harness, registers this object as a [DiceStorageSchema] bean. + * That one registration makes the stores' MERGEs race-free, because Drivine ensures [specs] from it, + * and tells [DiceOwnedSchema] these labels are dice's own, which keeps governance from reporting its + * own bookkeeping as domain drift. */ -object MetamodelSchema { +object MetamodelSchema : DiceStorageSchema { /** * Every MERGE these stores perform needs its key to be unique, because a MERGE is race-free @@ -38,7 +37,7 @@ object MetamodelSchema { * a position impossible to store, so a lost counter update fails with a constraint violation the * caller can retry. */ - fun specs(): List = listOf( + override fun specs(): List = listOf( // Version stamps -- see DrivineMetamodelVersionStore. UniquenessConstraintSpec(label = "MetamodelVersion", properties = listOf("schemaName", "contentHash")), UniquenessConstraintSpec(label = "MetamodelSchemaCounter", property = "schemaName"), diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt index 6a3d33ec..6e9dafea 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceOwnedSchemaTest.kt @@ -23,29 +23,34 @@ import org.junit.jupiter.api.assertThrows /** * Unit tests for [DiceOwnedSchema]: no database, just the shapes it reads out of the storage - * definitions. + * definitions it was given. * * These pin the derivation itself. `DrivineObservedSchemaSourceIntegrationTest` pins the thing that * matters downstream — nodes dice really wrote stay out of an observation, and domain nodes sharing * a label stay in it — which is what would catch a shape that drifted away from the writers. + * `DiceStorageSchemaRegistrationTest` pins that every dice store on the classpath is in the list an + * application passes to [DiceOwnedSchema.of]. */ class DiceOwnedSchemaTest { + /** The registered set a graph-backed application wiring every dice store would hand over. */ + private val owned = DiceOwnedSchema.of(listOf(MetamodelSchema, CollectorTraceSchema, LineageSchema)) + @Test fun `a source's shape holds every property dice writes on one`() { // The reviewer's case in one line: `key` alone is dice's uniqueness key, and a host is free // to key its own Source type the same way. `kind` is what dice also always writes. - assertEquals(listOf("key", "kind"), DiceOwnedSchema.NODE_SHAPES["Source"]) + assertEquals(listOf("key", "kind"), owned.nodeShapes["Source"]) } @Test fun `a mention's shape is what an extractor always records`() { - assertEquals(listOf("id", "span", "type", "role"), DiceOwnedSchema.NODE_SHAPES["Mention"]) + assertEquals(listOf("id", "span", "type", "role"), owned.nodeShapes["Mention"]) } @Test fun `a property dice can leave out stays out of the shape`() { - val proposition = DiceOwnedSchema.NODE_SHAPES.getValue("Proposition") + val proposition = owned.nodeShapes.getValue("Proposition") assertEquals(listOf("id", "contextId", "text", "confidence", "created"), proposition) assertFalse(proposition.contains("status"), "status carries a default, so an older node can lack it") @@ -60,23 +65,69 @@ class DiceOwnedSchemaTest { // version node takes its sequence in the same statement that creates it. assertEquals( listOf("schemaName", "contentHash", "sequence"), - DiceOwnedSchema.NODE_SHAPES["MetamodelVersion"], + owned.nodeShapes["MetamodelVersion"], ) assertEquals( listOf("propositionId", "runId", "target"), - DiceOwnedSchema.NODE_SHAPES["ProjectionRecord"], + owned.nodeShapes["ProjectionRecord"], + ) + assertEquals(listOf("runId"), owned.nodeShapes["CollectorTraceRun"]) + } + + @Test + fun `the core proposition store is owned whatever an application registered`() { + // The four node fragments are how the observer reads propositions and mentions at all, so + // they are in every ownership set, including one built from no registrations. + val bare = DiceOwnedSchema.of(emptyList()) + + assertEquals( + setOf("Proposition", "Mention", "Source", "ProcessedChunk"), + bare.labels, + ) + assertEquals(setOf("HAS_MENTION", "DERIVED_FROM"), bare.bookkeepingRelationshipTypes) + } + + @Test + fun `a store the application registered is owned, and one it did not is not`() { + // The whole fix in one assertion pair. Ownership follows the registered list, so a schema + // this application wired is dice's own, and the same schema left out of another + // application's wiring stays visible to that application's observation. + val withCollectorTrace = DiceOwnedSchema.of(listOf(CollectorTraceSchema)) + val withoutCollectorTrace = DiceOwnedSchema.of(listOf(MetamodelSchema)) + + CollectorTraceSchema.LABELS.forEach { label -> + assertTrue( + withCollectorTrace.nodeShapes.containsKey(label), + "'$label' was registered and carries no ownership shape", + ) + assertFalse( + withoutCollectorTrace.nodeShapes.containsKey(label), + "'$label' was never registered, so this application knows nothing about it", + ) + } + } + + @Test + fun `a registered store contributes its bookkeeping edges too`() { + assertEquals( + setOf("HAS_MENTION", "DERIVED_FROM", "SCORED", "RETIRED_IN"), + DiceOwnedSchema.of(listOf(CollectorTraceSchema)).bookkeepingRelationshipTypes, + ) + assertEquals( + setOf("HAS_MENTION", "DERIVED_FROM"), + DiceOwnedSchema.of(listOf(MetamodelSchema)).bookkeepingRelationshipTypes, + "an unregistered trace store's edges are nothing this application can vouch for", ) - assertEquals(listOf("runId"), DiceOwnedSchema.NODE_SHAPES["CollectorTraceRun"]) } @Test - fun `every label a dice store declares has a shape`() { - // The derivation has to reach all three schema objects. A label declared by a store and - // missing here would be reported as domain drift on every whole-graph check. + fun `every label a registered store declares has a shape`() { + // A label a registered store declares and this map misses would be reported as domain drift + // on every whole-graph check, which is the failure this derivation exists to rule out. (CollectorTraceSchema.LABELS + MetamodelSchema.LABELS + LineageSchema.LABELS).forEach { label -> assertTrue( - DiceOwnedSchema.NODE_SHAPES.containsKey(label), - "'$label' is written by a dice store and carries no ownership shape", + owned.nodeShapes.containsKey(label), + "'$label' is written by a registered dice store and carries no ownership shape", ) } } @@ -85,13 +136,13 @@ class DiceOwnedSchemaTest { fun `the ownership predicate asks for every property of the shape`() { assertEquals( "s.key IS NOT NULL AND s.kind IS NOT NULL", - DiceOwnedSchema.ownedNodePredicate("s", "Source"), + owned.ownedNodePredicate("s", "Source"), ) } @Test fun `a label dice never writes has no ownership predicate`() { - val thrown = assertThrows { DiceOwnedSchema.ownedNodePredicate("n", "Ghost") } + val thrown = assertThrows { owned.ownedNodePredicate("n", "Ghost") } assertTrue(thrown.message!!.contains("Ghost"), "got ${thrown.message}") } diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceStorageSchemaRegistrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceStorageSchemaRegistrationTest.kt new file mode 100644 index 00000000..6eb08dd5 --- /dev/null +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DiceStorageSchemaRegistrationTest.kt @@ -0,0 +1,130 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.dice.storage + +import org.drivine.annotation.NodeFragment +import org.drivine.schema.SchemaCatalog +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider +import org.springframework.core.type.filter.AnnotationTypeFilter +import org.springframework.core.type.filter.AssignableTypeFilter +import org.springframework.core.type.filter.TypeFilter +import org.springframework.test.context.DynamicPropertyRegistry +import org.springframework.test.context.DynamicPropertySource + +/** + * The guard that would have caught the union-branch defect. + * + * A whole-graph drift check hides dice's own storage, and it works out what that is from the + * [DiceStorageSchema] beans the application registered. A dice store nobody registered therefore + * reports its own nodes as domain drift, forever, on every check. The guard this replaces was + * written against the very hand-kept list of schema objects that was missing one, so it could never + * fail: it compared the list with itself. + * + * These assertions compare two independent things — what the classpath holds, and what the running + * Spring context registered — so a schema object that exists and is wired nowhere fails the build in + * the slice that adds it, with no list here to remember to update. + * + * [TestApplication] is this repository's only wiring of a [DrivineObservedSchemaSource]. A host wires + * it the same way, and the autoconfigure module's own + * `DiceStorageSchemaRegistrationTest` holds its graph backend to the second rule below. + */ +@SpringBootTest(classes = [TestApplication::class]) +class DiceStorageSchemaRegistrationTest { + + companion object { + @JvmStatic + @DynamicPropertySource + fun neo4jProperties(registry: DynamicPropertyRegistry) = Neo4jTestContainer.registerProperties(registry) + } + + @Autowired + private lateinit var registeredSchemas: List + + @Autowired + private lateinit var registeredCatalogs: List + + @Test + fun `every dice storage schema on the classpath is registered`() { + val onClasspath = diceStorageSchemasOnClasspath().map { it::class }.toSet() + val registered = registeredSchemas.map { it::class }.toSet() + + assertEquals( + onClasspath, + registered, + "a dice storage schema that reaches no registration is invisible to the drift " + + "exclusion, so every label it declares is reported as domain drift once its store " + + "writes anything. Register it beside the store it belongs to. Missing: " + + "${onClasspath - registered}; registered and absent from the classpath scan: " + + "${registered - onClasspath}", + ) + } + + @Test + fun `no dice store's schema reaches the database without contributing to ownership`() { + // The other direction, and the one that catches a hand-written + // `SchemaCatalog.of(SomeSchema.specs())`. Such a catalog creates the store's constraints, + // which mints its labels in the database, while leaving the store out of the ownership the + // observer subtracts. Deriving every catalog through `diceStorageCatalog` closes it. + val registered = registeredSchemas.map { it::class }.toSet() + val declaredSpecs = registeredCatalogs.flatMap { catalog -> catalog.items }.toSet() + + diceStorageSchemasOnClasspath() + .filter { schema -> schema::class !in registered } + .forEach { unregistered -> + val leaked = unregistered.specs().filter { spec -> spec in declaredSpecs } + assertTrue( + leaked.isEmpty(), + "${unregistered::class.simpleName} is ensured against the database without being " + + "registered as a DiceStorageSchema, so its labels drift: $leaked", + ) + } + } + + @Test + fun `every dice node fragment is part of the core ownership set`() { + // The fragments dice persists directly are ownership no registration can switch off, because + // the observer reads propositions and mentions through their shapes. A new one has to join + // `DiceOwnedSchema.CORE_NODE_FRAGMENTS` for its label to be recognised as dice's own. + val onClasspath = scan("com.embabel.dice.storage.model", AnnotationTypeFilter(NodeFragment::class.java)) + .map { it.kotlin } + .toSet() + + assertEquals( + onClasspath, + DiceOwnedSchema.CORE_NODE_FRAGMENTS.toSet(), + "a @NodeFragment dice writes carries no ownership shape, so its label is reported as " + + "domain drift on every whole-graph check", + ) + } + + /** Every `DiceStorageSchema` singleton the storage classpath holds, main and test alike. */ + private fun diceStorageSchemasOnClasspath(): List = + scan("com.embabel.dice.storage", AssignableTypeFilter(DiceStorageSchema::class.java)) + .mapNotNull { candidate -> candidate.kotlin.objectInstance as DiceStorageSchema? } + + private fun scan(basePackage: String, filter: TypeFilter): List> { + val scanner = ClassPathScanningCandidateComponentProvider(false) + scanner.addIncludeFilter(filter) + return scanner.findCandidateComponents(basePackage) + .mapNotNull { definition -> definition.beanClassName } + .map { name -> Class.forName(name) } + } +} diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt index a1e457f3..91faff57 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt @@ -20,11 +20,11 @@ import com.embabel.dice.common.DiceMetadataKeys import com.embabel.dice.metamodel.DeclaredSchema import com.embabel.dice.metamodel.DeclaredSchemaSource import com.embabel.dice.metamodel.DriftCheckRunner -import com.embabel.dice.metamodel.DriftSweepCapable +import com.embabel.dice.spi.DriftSweepCapable import com.embabel.dice.metamodel.MetamodelVersion import com.embabel.dice.metamodel.support.DefaultDriftCheckRunner -import com.embabel.dice.metamodel.support.MentionTypeDriftQuarantinePolicy -import com.embabel.dice.metamodel.support.PropositionStoreDriftSweep +import com.embabel.dice.spi.MentionTypeDriftQuarantinePolicy +import com.embabel.dice.spi.PropositionStoreDriftSweep import com.embabel.dice.metamodel.support.StructuralMetamodelDiffer import com.embabel.dice.proposition.EntityMention import com.embabel.dice.proposition.MentionRole @@ -186,7 +186,7 @@ class DrivineDriftCheckIntegrationTest { // quarantine reason. val reloaded = repository.findById(stranded.id) assertNotNull(reloaded) - assertEquals(PropositionStatus.STALE, reloaded!!.status) + assertEquals(PropositionStatus.QUARANTINED, reloaded!!.status) val reason = reloaded.metadata[DiceMetadataKeys.QUARANTINE_REASON] as? String assertNotNull(reason, "quarantine must say why; metadata was ${reloaded.metadata}") assertTrue(reason!!.contains("Ghost"), "the reason must name the drifted type, but was: $reason") @@ -314,7 +314,7 @@ class DrivineDriftCheckIntegrationTest { val swept = sweep.sweep(result.quarantineDiff, policy, contextId) assertEquals(listOf(stranded.id), swept.quarantined.map { it.proposition.id }) - assertEquals(PropositionStatus.STALE, repository.findById(stranded.id)!!.status) + assertEquals(PropositionStatus.QUARANTINED, repository.findById(stranded.id)!!.status) } @Test @@ -359,7 +359,7 @@ class DrivineDriftCheckIntegrationTest { assertEquals(setOf("Agent"), result.report.driftedEntityTypes) val swept = sweep.sweep(result.quarantineDiff, policy, contextId) assertEquals(listOf(stranded.id), swept.quarantined.map { it.proposition.id }) - assertEquals(PropositionStatus.STALE, repository.findById(stranded.id)!!.status) + assertEquals(PropositionStatus.QUARANTINED, repository.findById(stranded.id)!!.status) } /** A projected entity node carrying a governed `Person`'s whole label hierarchy. */ diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt index 1c2db9a8..57d9cb60 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt @@ -74,6 +74,10 @@ class DrivineObservedSchemaSourceIntegrationTest { @Autowired private lateinit var chunkHistoryStore: DrivineChunkHistoryStore + /** What dice owns in this context, derived from the schemas [TestApplication] registered. */ + @Autowired + private lateinit var ownedSchema: DiceOwnedSchema + private val tenantA = ContextId("tenant-a") private val tenantB = ContextId("tenant-b") @@ -235,8 +239,8 @@ class DrivineObservedSchemaSourceIntegrationTest { val observed = source.observe() assertTrue( - observed.entityTypeNames.none { it in DiceOwnedSchema.LABELS }, - "bookkeeping leaked into the observation: ${observed.entityTypeNames intersect DiceOwnedSchema.LABELS}", + observed.entityTypeNames.none { it in ownedSchema.labels }, + "bookkeeping leaked into the observation: ${observed.entityTypeNames intersect ownedSchema.labels}", ) } @@ -292,9 +296,9 @@ class DrivineObservedSchemaSourceIntegrationTest { val observed = source.observe() assertTrue( - observed.relationshipTypeNames.none { it in DICE_BOOKKEEPING_RELATIONSHIP_TYPES }, + observed.relationshipTypeNames.none { it in ownedSchema.bookkeepingRelationshipTypes }, "bookkeeping edges leaked: " + - "${observed.relationshipTypeNames intersect DICE_BOOKKEEPING_RELATIONSHIP_TYPES}", + "${observed.relationshipTypeNames intersect ownedSchema.bookkeepingRelationshipTypes}", ) } @@ -439,17 +443,89 @@ class DrivineObservedSchemaSourceIntegrationTest { ) } + // ---- Ownership follows what the application registered ---- + @Test - fun `every label a dice store declares has an ownership shape`() { - // The shapes are derived from these same schema objects, so this is a pin on the derivation - // reaching all of them: a label declared by a store and missing here would stop being - // excluded. - (CollectorTraceSchema.LABELS + MetamodelSchema.LABELS + LineageSchema.LABELS).forEach { label -> - assertTrue( - DiceOwnedSchema.NODE_SHAPES.containsKey(label), - "'$label' is written by a dice store and carries no ownership shape", - ) - } + fun `a registered store's label with no nodes is not observed`() { + // The union-branch case exactly: a dice store declared at startup whose records nothing has + // written yet. Its constraint mints the label in `db.labels()` on an empty graph, and the + // old observation reported all such labels as domain drift on the host's very first check. + assertTrue( + rawLabels().contains(ProbeStoreSchema.PROBE_RECORD), + "precondition: the constraint alone must put the label in the catalogue, but was ${rawLabels()}", + ) + assertEquals(0L, nodeCount(ProbeStoreSchema.PROBE_RECORD), "precondition: no probe node exists") + + val observed = source.observe() + + assertFalse( + observed.entityTypeNames.contains(ProbeStoreSchema.PROBE_RECORD), + "a registered store's empty label was reported as drift; got ${observed.entityTypeNames}", + ) + } + + @Test + fun `a registered store's own nodes are excluded from drift`() { + // The half a node-bearing rule alone cannot carry: once real records exist, only ownership + // keeps them out. `ProbeStoreSchema` is declared in the test wiring and named nowhere in + // `DiceOwnedSchema`, so passing this proves the exclusion is derived from the registration. + persistenceManager.execute( + QuerySpecification.withStatement("CREATE (:ProbeRecord {probeId: 'probe-1'})"), + ) + assertEquals(1L, nodeCount(ProbeStoreSchema.PROBE_RECORD), "precondition: a probe node exists") + + val observed = source.observe() + + assertFalse( + observed.entityTypeNames.contains(ProbeStoreSchema.PROBE_RECORD), + "a registered store's own nodes were reported as domain drift; got ${observed.entityTypeNames}", + ) + } + + @Test + fun `an unregistered label carrying nodes is still drift`() { + // The acceptance boundary. Deriving ownership from registrations must widen nothing: a label + // no registered schema declares is domain data, and its nodes are exactly what a drift check + // exists to report. + persistenceManager.execute( + QuerySpecification.withStatement("CREATE (:UnregisteredRecord {recordId: 'r-1'})"), + ) + + val observed = source.observe() + + assertTrue( + observed.entityTypeNames.contains("UnregisteredRecord"), + "a label dice owns nothing of stopped being drift; got ${observed.entityTypeNames}", + ) + } + + @Test + fun `a label a constraint minted and no node wears is not observed`() { + // The node-bearing rule on its own, held apart from ownership: this label belongs to no dice + // store, so nothing subtracts it. `db.labels()` reports it because a constraint names it, + // and constraint DDL is schema machinery an application declared. An observation reports + // what data the graph holds, and there is no data here. + persistenceManager.execute( + QuerySpecification.withStatement( + "CREATE CONSTRAINT constraint_only_probe IF NOT EXISTS " + + "FOR (n:ConstraintOnlyDomainLabel) REQUIRE n.id IS UNIQUE", + ), + ) + assertTrue( + rawLabels().contains("ConstraintOnlyDomainLabel"), + "precondition: the constraint alone must put the label in the catalogue, but was ${rawLabels()}", + ) + + val observed = source.observe() + + assertFalse( + observed.entityTypeNames.contains("ConstraintOnlyDomainLabel"), + "a label no node wears was reported as drift; got ${observed.entityTypeNames}", + ) + + persistenceManager.execute( + QuerySpecification.withStatement("DROP CONSTRAINT constraint_only_probe IF EXISTS"), + ) } // ---- helpers ---- @@ -545,6 +621,11 @@ class DrivineObservedSchemaSourceIntegrationTest { .filterNotNull() .toSet() + /** How many nodes wear a label, straight from the database. */ + private fun nodeCount(label: String): Long = persistenceManager.maybeGetOne( + QuerySpecification.withStatement("MATCH (n:$label) RETURN count(n) AS c").transform(Long::class.java), + ) ?: 0L + private fun governanceNodeCount(): Int = MetamodelSchema.LABELS.count { label -> ( persistenceManager.maybeGetOne( diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt index fd30c24c..29d3734c 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/TestApplication.kt @@ -24,6 +24,7 @@ import org.drivine.manager.GraphObjectManagerFactory import org.drivine.manager.PersistenceManager import org.drivine.manager.PersistenceManagerFactory import org.drivine.schema.SchemaCatalog +import org.drivine.schema.SchemaItemSpec import org.drivine.schema.SimilarityFunction import org.drivine.schema.UniquenessConstraintSpec import org.drivine.schema.VectorIndexSpec @@ -133,7 +134,7 @@ open class TestApplication { * same list. A host wires it the same way. */ @Bean - open fun lineageSchema(): SchemaCatalog = SchemaCatalog.of(LineageSchema.specs()) + open fun lineageSchema(): DiceStorageSchema = LineageSchema @Bean open fun projectionRecordStore( @@ -146,7 +147,7 @@ open class TestApplication { ): DrivineCollectorRecordStore = DrivineCollectorRecordStore(persistenceManager) @Bean - open fun collectorTraceSchema(): SchemaCatalog = SchemaCatalog.of(CollectorTraceSchema.specs()) + open fun collectorTraceSchema(): DiceStorageSchema = CollectorTraceSchema @Bean open fun collectorTraceStore( @@ -169,11 +170,35 @@ open class TestApplication { * The two `sequence` constraints back the ordering. They make two records of one schema sharing * a position impossible to store, so a lost counter update fails with a constraint violation the * caller can retry. `DrivineMetamodelVersionStoreIntegrationTest` and - * `DrivineDriftReportStoreIntegrationTest` pin them. The list lives in [MetamodelSchema], which - * keeps the constraints and the label list the observed-schema source excludes in one place. + * `DrivineDriftReportStoreIntegrationTest` pin them. The list lives in [MetamodelSchema], and + * registering it here is also what tells the observed-schema source these labels are dice's own. */ @Bean - open fun metamodelSchema(): SchemaCatalog = SchemaCatalog.of(MetamodelSchema.specs()) + open fun metamodelSchema(): DiceStorageSchema = MetamodelSchema + + /** + * A store this harness declares and never writes to, standing in for a dice store some other + * slice adds. It reaches the drift exclusion purely by being registered below, with no edit to + * [DiceOwnedSchema] or [DrivineObservedSchemaSource] — which is the property the union of two + * branches broke when the exclusion was a hand-written list of schema objects. + */ + @Bean + open fun probeStoreSchema(): DiceStorageSchema = ProbeStoreSchema + + /** + * Every dice storage schema this context registered, gathered into the one catalog Drivine + * ensures on startup. The same bean list builds [diceOwnedSchema], which is the property the + * whole design rests on: a store's constraints and its drift exclusion cannot come apart, + * because one registration produces both. + */ + @Bean + open fun diceStorageSchemaCatalog(schemas: List): SchemaCatalog = + diceStorageCatalog(schemas) + + /** What dice owns in this context, derived from the same registrations. */ + @Bean + open fun diceOwnedSchema(schemas: List): DiceOwnedSchema = + DiceOwnedSchema.of(schemas) @Bean open fun metamodelClock(): PinnableClock = PinnableClock() @@ -197,5 +222,26 @@ open class TestApplication { @Bean open fun observedSchemaSource( persistenceManager: PersistenceManager, - ): DrivineObservedSchemaSource = DrivineObservedSchemaSource(persistenceManager) + ownedSchema: DiceOwnedSchema, + ): DrivineObservedSchemaSource = DrivineObservedSchemaSource(persistenceManager, ownedSchema) +} + +/** + * A test-only dice store schema, standing in for one a later slice adds. + * + * Nothing writes `(:ProbeRecord)` nodes in ordinary running, so the label exists in the database + * through this constraint alone — which is exactly the state that used to be reported as permanent + * whole-graph drift. `DrivineObservedSchemaSourceIntegrationTest` covers both halves: the label with + * no nodes is no observation at all, and once a node wears it, ownership derived from this + * registration keeps it out of drift. + */ +object ProbeStoreSchema : DiceStorageSchema { + + const val PROBE_RECORD: String = "ProbeRecord" + + override val bookkeepingRelationshipTypes: Set = setOf("PROBED_BY") + + override fun specs(): List = listOf( + UniquenessConstraintSpec(label = PROBE_RECORD, property = "probeId"), + ) } From a8eeb76baa960f83d21ae0549d2b13193da3df0e Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Wed, 2 Sep 2026 05:51:31 -0400 Subject: [PATCH 07/10] State the schema-registration trade and steady the guard fixture Registering a DiceStorageSchema bean now says on the interface what it does: the schema's labels and relationship types become storage bookkeeping the drift observation stops reporting, which is right for a store's own nodes and wrong for domain data, whose home is the DataDictionary. The guard fixture supplies a transaction manager so a store bean that manages its own transactions can wire in the scanned context, and a stale KDoc link follows the nodeShapes rename. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../autoconfigure/DiceStorageSchemaRegistrationTest.kt | 5 +++++ .../kotlin/com/embabel/dice/storage/DiceStorageSchema.kt | 6 ++++++ .../storage/DrivineObservedSchemaSourceIntegrationTest.kt | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt b/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt index 336fc2d3..3e589b54 100644 --- a/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt +++ b/dice-storage-autoconfigure/src/test/kotlin/com/embabel/dice/storage/autoconfigure/DiceStorageSchemaRegistrationTest.kt @@ -21,6 +21,7 @@ import com.embabel.dice.storage.LineageSchema import org.assertj.core.api.Assertions.assertThat import org.drivine.manager.GraphObjectManager import org.drivine.manager.PersistenceManager +import org.springframework.transaction.PlatformTransactionManager import org.drivine.schema.SchemaCatalog import org.drivine.schema.SchemaItemSpec import org.junit.jupiter.api.Test @@ -123,6 +124,10 @@ class DiceStorageSchemaRegistrationTest { @Bean open fun graphObjectManager(): GraphObjectManager = mock() + + /** Store beans that manage their own transactions ask for one of these at wiring time. */ + @Bean + open fun transactionManager(): PlatformTransactionManager = mock() } /** Every `DiceStorageSchema` singleton on this module's classpath. */ diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt index 9fef5361..8e09da14 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt @@ -31,6 +31,12 @@ import org.drivine.schema.SchemaItemSpec * Adding a store is therefore a two-file job in the slice that adds it: declare its schema object * beside the store, and register it where the application wires its stores. Nothing in the drift * machinery has to hear about it. + * + * Registering a bean of this type is a declaration with a consequence: every label and + * relationship type the schema names is storage bookkeeping, and the drift observation stops + * reporting it. That is the point for a store's own nodes and exactly wrong for domain data — a + * domain type belongs in the [DataDictionary], where governance can see it. A host + * that registers its own schema object takes on that trade knowingly. */ interface DiceStorageSchema { diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt index 57d9cb60..200ae486 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSourceIntegrationTest.kt @@ -533,7 +533,7 @@ class DrivineObservedSchemaSourceIntegrationTest { /** * A proposition node carrying every property dice's own writer always writes, which is how the * observer recognises it as dice's own where a domain node shares the label. The list is - * [DiceOwnedSchema.NODE_SHAPES] for `Proposition`, read off `PropositionNode`'s required + * [DiceOwnedSchema.nodeShapes] for `Proposition`, read off `PropositionNode`'s required * constructor parameters. */ private fun writeProposition(id: String, contextId: ContextId) { From 848bb6961128085610e6598dda9c095aeb934576 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:33:35 -0400 Subject: [PATCH 08/10] Mark the storage governance surface experimental Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- dice-storage/pom.xml | 11 +++++++++++ .../com/embabel/dice/storage/DiceOwnedSchema.kt | 3 +++ .../com/embabel/dice/storage/DiceStorageSchema.kt | 3 +++ .../embabel/dice/storage/DrivineDriftReportStore.kt | 2 ++ .../dice/storage/DrivineObservedSchemaSource.kt | 2 ++ .../kotlin/com/embabel/dice/storage/LineageSchema.kt | 3 +++ .../com/embabel/dice/storage/MetamodelSchema.kt | 3 +++ 7 files changed, 27 insertions(+) diff --git a/dice-storage/pom.xml b/dice-storage/pom.xml index c461dc10..fc8bc82a 100644 --- a/dice-storage/pom.xml +++ b/dice-storage/pom.xml @@ -76,6 +76,17 @@ slf4j-api + + + org.jetbrains + annotations + 26.0.2 + provided + + org.springframework.boot diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt index 119f3814..f9d92540 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceOwnedSchema.kt @@ -15,6 +15,8 @@ */ package com.embabel.dice.storage +import org.jetbrains.annotations.ApiStatus + import com.embabel.dice.storage.model.Mention import com.embabel.dice.storage.model.ProcessedChunkNode import com.embabel.dice.storage.model.PropositionNode @@ -98,6 +100,7 @@ import kotlin.reflect.full.primaryConstructor * carrying that label as dice's own. A node missing any of them was written by somebody else. * @property bookkeepingRelationshipTypes Every relationship type dice writes for itself. */ +@ApiStatus.Experimental class DiceOwnedSchema private constructor( val nodeShapes: Map>, val bookkeepingRelationshipTypes: Set, diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt index 8e09da14..0d563d22 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DiceStorageSchema.kt @@ -15,6 +15,8 @@ */ package com.embabel.dice.storage +import org.jetbrains.annotations.ApiStatus + import org.drivine.schema.SchemaCatalog import org.drivine.schema.SchemaItemSpec @@ -38,6 +40,7 @@ import org.drivine.schema.SchemaItemSpec * domain type belongs in the [DataDictionary], where governance can see it. A host * that registers its own schema object takes on that trade knowingly. */ +@ApiStatus.Experimental interface DiceStorageSchema { /** The constraints and indexes this store's writes depend on. */ diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt index 7d5ff385..48034a79 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineDriftReportStore.kt @@ -15,6 +15,8 @@ */ package com.embabel.dice.storage +import org.jetbrains.annotations.ApiStatus + import com.embabel.agent.core.ContextId import com.embabel.dice.metamodel.DriftReport import com.embabel.dice.metamodel.DriftReportStore diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt index 5ab44771..f260705a 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivineObservedSchemaSource.kt @@ -15,6 +15,8 @@ */ package com.embabel.dice.storage +import org.jetbrains.annotations.ApiStatus + import com.embabel.agent.core.ContextId import com.embabel.dice.metamodel.ObservedSchema import com.embabel.dice.metamodel.ObservedSchemaSource diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt index 6914a41f..28fa0f23 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/LineageSchema.kt @@ -15,6 +15,8 @@ */ package com.embabel.dice.storage +import org.jetbrains.annotations.ApiStatus + import org.drivine.schema.RangeIndexSpec import org.drivine.schema.SchemaItemSpec import org.drivine.schema.UniquenessConstraintSpec @@ -35,6 +37,7 @@ import org.drivine.schema.UniquenessConstraintSpec * Cypher and nowhere else can lose its constraint, or stop matching the shape an observation * recognises, with nothing failing to say so. */ +@ApiStatus.Experimental object LineageSchema : DiceStorageSchema { /** One node per projection outcome. */ diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt index a6d68d2f..6d266a75 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/MetamodelSchema.kt @@ -15,6 +15,8 @@ */ package com.embabel.dice.storage +import org.jetbrains.annotations.ApiStatus + import org.drivine.schema.SchemaItemSpec import org.drivine.schema.UniquenessConstraintSpec @@ -26,6 +28,7 @@ import org.drivine.schema.UniquenessConstraintSpec * and tells [DiceOwnedSchema] these labels are dice's own, which keeps governance from reporting its * own bookkeeping as domain drift. */ +@ApiStatus.Experimental object MetamodelSchema : DiceStorageSchema { /** From 18f1f57f8c0f96f14819d27e823e6d5e5c5c9b31 Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:33:13 -0400 Subject: [PATCH 09/10] Take the quarantine keys off the node on release The Drivine property bag is merge-only, so a release that dropped the two quarantine keys from the proposition's metadata left previousStatus and reason on the node. Saving a proposition now removes every metadata.* property the saved metadata no longer names, with one dynamic REMOVE after the node write. The drift integration test checks the keys are gone. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- .../storage/DrivinePropositionRepository.kt | 26 +++++++++++ .../DrivineDriftCheckIntegrationTest.kt | 46 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt index e62b0b48..b243313d 100644 --- a/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt +++ b/dice-storage/src/main/kotlin/com/embabel/dice/storage/DrivinePropositionRepository.kt @@ -368,10 +368,36 @@ class DrivinePropositionRepository( private fun doPersist(proposition: Proposition): Proposition { val embedding = embeddingFor(proposition) graphObjectManager.save(PropositionGraphMapper.toView(proposition, embedding), CascadeType.DELETE_ORPHAN) + dropMetadataKeysNotIn(proposition) appendProvenance(proposition.id, proposition.provenanceEntries) return proposition } + /** + * A saved proposition's metadata map is the whole map, the same way its status and text are the + * whole value. Drivine spreads the bag over `metadata.` node properties and only ever adds + * or overwrites, so a key the caller took out of the map (a released quarantine's + * `previousStatus` and reason, say) stayed on the node. This removes every `metadata.*` + * property the saved map no longer carries. Needs dynamic property removal (`REMOVE n[k]`), + * which Neo4j has had since 5.24. + */ + private fun dropMetadataKeysNotIn(proposition: Proposition) { + persistenceManager.execute( + QuerySpecification + .withStatement( + "MATCH (p:Proposition {id: \$id}) " + + "UNWIND [k IN keys(p) WHERE k STARTS WITH 'metadata.' AND NONE(x IN \$kept WHERE x = k)] AS k " + + "REMOVE p[k]", + ) + .bind( + mapOf( + "id" to proposition.id, + "kept" to proposition.metadata.keys.map { "metadata.$it" }, + ), + ), + ) + } + private fun embeddingFor(proposition: Proposition): List? = proposition.text.takeIf { it.isNotBlank() }?.let { embeddingService.embed(it).toList() } diff --git a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt index 91faff57..732d360a 100644 --- a/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt +++ b/dice-storage/src/test/kotlin/com/embabel/dice/storage/DrivineDriftCheckIntegrationTest.kt @@ -17,6 +17,7 @@ package com.embabel.dice.storage import com.embabel.agent.core.ContextId import com.embabel.dice.common.DiceMetadataKeys +import com.embabel.dice.spi.DriftQuarantineKeys import com.embabel.dice.metamodel.DeclaredSchema import com.embabel.dice.metamodel.DeclaredSchemaSource import com.embabel.dice.metamodel.DriftCheckRunner @@ -36,6 +37,7 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired @@ -192,6 +194,33 @@ class DrivineDriftCheckIntegrationTest { assertTrue(reason!!.contains("Ghost"), "the reason must name the drifted type, but was: $reason") } + @Test + fun `a release restores the prior status and takes both quarantine keys off the node`() { + val stranded = repository.save( + Proposition( + contextId = contextId, + text = "The ghost walks the halls", + mentions = listOf(EntityMention(span = "the ghost", type = "Ghost", role = MentionRole.SUBJECT)), + confidence = 0.9, + ), + ) + val result = runner.run(contextId) + sweep.sweep(result.quarantineDiff, policy, contextId) + assertEquals(setOf(previousStatusProperty, reasonProperty), quarantineProperties(stranded.id)) + + val released = sweep.releaseFromQuarantine(stranded.id) + + assertNotNull(released) + assertEquals(PropositionStatus.ACTIVE, released!!.status) + val reloaded = repository.findById(stranded.id) + assertEquals(PropositionStatus.ACTIVE, reloaded?.status) + assertNull(reloaded?.metadata?.get(DriftQuarantineKeys.PREVIOUS_STATUS)) + assertNull(reloaded?.metadata?.get(DiceMetadataKeys.QUARANTINE_REASON)) + // The node itself, read raw: the release's save must have taken the flattened properties off, + // since a metadata map that merely omits a key used to leave the old property in place. + assertEquals(emptySet(), quarantineProperties(stranded.id)) + } + @Test fun `a check records its report and touches no proposition`() { val untouched = repository.save( @@ -375,4 +404,21 @@ class DrivineDriftCheckIntegrationTest { ) .filterNotNull() .toSet() + + private val previousStatusProperty = "metadata.${DriftQuarantineKeys.PREVIOUS_STATUS}" + private val reasonProperty = "metadata.${DiceMetadataKeys.QUARANTINE_REASON}" + + /** The two flattened quarantine properties as they sit on the node itself, whichever are present. */ + private fun quarantineProperties(propositionId: String): Set = persistenceManager + .query( + QuerySpecification + .withStatement( + "MATCH (p:Proposition {id: \$id}) UNWIND keys(p) AS k " + + "WITH k WHERE k IN \$watched RETURN k", + ) + .bind(mapOf("id" to propositionId, "watched" to listOf(previousStatusProperty, reasonProperty))) + .transform(String::class.java), + ) + .filterNotNull() + .toSet() } From 8004df55c38c03dccf64b57f0ec6d7268401090f Mon Sep 17 00:00:00 2001 From: James Dunnam <7660553+jimador@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:33:27 -0400 Subject: [PATCH 10/10] Declare the annotations dependency once in dice-storage The version now comes from the root pom's dependencyManagement. Signed-off-by: James Dunnam <7660553+jimador@users.noreply.github.com> --- dice-storage/pom.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dice-storage/pom.xml b/dice-storage/pom.xml index fc8bc82a..b029ad5e 100644 --- a/dice-storage/pom.xml +++ b/dice-storage/pom.xml @@ -76,14 +76,10 @@ slf4j-api - + org.jetbrains annotations - 26.0.2 provided