feat(stores): run-scoped knowledge stores with lineage-chain reads, and the claim-evidence contract

src/claim-evidence.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from 'vitest'
+import { assertGradeableEvidence, UncheckableClaimError, verdictFor } from './claim-evidence'
+
+describe('assertGradeableEvidence', () => {
+  it('refuses rung 4+ without a check — a self-grade must fail at record time', () => {
+    expect(() => assertGradeableEvidence({ rung: 4 })).toThrow(UncheckableClaimError)
+    expect(() => assertGradeableEvidence({ rung: 5 })).toThrow(UncheckableClaimError)
+  })
+  it('passes rung 4+ with a check, and any rung below the threshold', () => {
+    expect(assertGradeableEvidence({ rung: 4, check: 'true' }).rung).toBe(4)
+    expect(assertGradeableEvidence({ rung: 3 }).rung).toBe(3)
+  })
+})
+
+describe('verdictFor — the calibration cases that were graded wrong before this lattice', () => {
+  it('a silent assert is not a contradiction', () => {
+    // exit 0, prints nothing: the expectation can never appear, and grading this 'contradicted'
+    // once flunked three TRUE claims on first contact with real data.
+    expect(
+      verdictFor({ rung: 4, check: 'x', expect: 'True' }, { exitCode: 0, stdout: '', stderr: '' }),
+    ).toBe('silent-check')
+  })
+  it('a printed decisive value verifies', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: 'x', expect: '2983' },
+        { exitCode: 0, stdout: 'roots_checked=2983', stderr: '' },
+      ),
+    ).toBe('verified')
+  })
+  it('a missing input blames the environment, not the claim', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: 'x' },
+        { exitCode: 1, stdout: '', stderr: 'FileNotFoundError: k3.json' },
+      ),
+    ).toBe('unrunnable')
+  })
+  it('a passing command printing the wrong value is a real contradiction', () => {
+    expect(
+      verdictFor(
+        { rung: 4, check: 'x', expect: '7' },
+        { exitCode: 0, stdout: 'value=8', stderr: '' },
+      ),
+    ).toBe('contradicted')
+  })
+  it('rung 4+ with no check is uncheckable regardless of execution', () => {
+    expect(verdictFor({ rung: 4 }, null)).toBe('uncheckable')
+  })
+  it('a check that could not execute at all is unrunnable', () => {
+    expect(verdictFor({ rung: 4, check: 'x' }, null)).toBe('unrunnable')
+  })
+})

src/claim-evidence.ts

@@ -0,0 +1,116 @@
+/**
+ * The evidence contract for claims that machines, not authors, get to grade.
+ *
+ * Grown in a discovery campaign that learned each rule by paying for it. Self-reported
+ * verification is a dead channel: an entire run's twenty-eight true claims scored zero because
+ * nothing carried a check a grader could re-execute, and a sibling lab's cells once produced six
+ * false certifications in seventeen deliveries without a single agent lying — the format betrayed
+ * them. The countermeasure is mechanical: a claim above a threshold must carry the command that
+ * re-establishes it, and an independent grader re-runs that command blind.
+ *
+ * The verdict lattice below is calibrated, not designed: its first contact with real claims
+ * graded three TRUE results as refuted, because a silent assert exits zero printing nothing and
+ * the expectation could never match empty output. Every distinction here exists because collapsing
+ * it blamed the wrong party — a claim for its environment, or an author for their formatting.
+ */
+
+/**
+ * How far up the ladder a claim's evidence actually reaches. The rungs are five different
+ * statements and only the top two are verification:
+ *
+ *   1 — it parses / exists          3 — its tests pass         5 — independently re-derived
+ *   2 — it imports / loads          4 — reproduces a claimed value
+ *
+ * Reporting a low rung in the vocabulary of a high one is the most expensive error available to a
+ * knowledge system: it is indistinguishable from success and propagates as settled provenance.
+ */
+export type EvidenceRung = 1 | 2 | 3 | 4 | 5
+
+/** The rung at and above which a claim must be machine-checkable to be recorded at that rung. */
+export const CHECKABLE_RUNG_THRESHOLD = 4
+
+export interface ClaimEvidence {
+  rung: EvidenceRung
+  /**
+   * A shell command that re-establishes the claim when run by a grader the author cannot edit,
+   * from a working directory the author must not assume. Required at rung 4 and above.
+   */
+  check?: string
+  /**
+   * A substring the check's stdout must contain — the decisive value, printed. A check that
+   * exits zero but prints nothing cannot confirm a value; authors should print the number.
+   */
+  expect?: string
+  /** Where the artifact backing the claim lives, for humans following the trail. */
+  evidencePath?: string
+}
+
+export class UncheckableClaimError extends Error {
+  constructor(rung: EvidenceRung) {
+    super(
+      `a claim at rung ${rung} asserts "a command reproduces a value" and must carry that ` +
+        `command (evidence.check); without one it is a self-grade, which is rung 3 at most`,
+    )
+    this.name = 'UncheckableClaimError'
+  }
+}
+
+/**
+ * Refuse the evidence shapes that made self-grading possible. Call at record time, not grade
+ * time: by grading it is too late — the ungradeable claim has already circulated as verified.
+ */
+export function assertGradeableEvidence(evidence: ClaimEvidence): ClaimEvidence {
+  if (evidence.rung >= CHECKABLE_RUNG_THRESHOLD && !evidence.check) {
+    throw new UncheckableClaimError(evidence.rung)
+  }
+  return evidence
+}
+
+/**
+ * What a grader may conclude from re-executing a claim's check.
+ *
+ *   verified      exit 0, and the expectation (if any) appears in output
+ *   silent-check  exit 0, expectation given, output empty — the check passed but proves nothing
+ *                 about the expected value; the author should make the check PRINT it
+ *   contradicted  the check ran and refuted the claim: nonzero exit, or non-empty output that
+ *                 lacks the expectation
+ *   unrunnable    the check itself could not execute (missing input, missing module) — an
+ *                 environment verdict, never a claim verdict
+ *   uncheckable   rung demanded a check and none was recorded — a self-grade, counted against
+ */
+export type ClaimVerdict =
+  | 'verified'
+  | 'silent-check'
+  | 'contradicted'
+  | 'unrunnable'
+  | 'uncheckable'
+
+/** The error signatures that mean the check could not run, as opposed to ran and failed. */
+const UNRUNNABLE_SIGNATURES =
+  /No such file|FileNotFoundError|ModuleNotFoundError|command not found|ENOENT/
+
+export interface CheckExecution {
+  exitCode: number
+  stdout: string
+  stderr: string
+}
+
+/**
+ * The calibrated verdict function, pure so every grader shares one semantics. Callers execute the
+ * check however their environment requires and pass the observation; this function only judges.
+ */
+export function verdictFor(
+  evidence: Pick<ClaimEvidence, 'rung' | 'check' | 'expect'>,
+  execution: CheckExecution | null,
+): ClaimVerdict {
+  if (evidence.rung >= CHECKABLE_RUNG_THRESHOLD && !evidence.check) return 'uncheckable'
+  if (!execution) return 'unrunnable'
+  const output = `${execution.stdout}\n${execution.stderr}`.trim()
+  if (execution.exitCode !== 0) {
+    return UNRUNNABLE_SIGNATURES.test(output) ? 'unrunnable' : 'contradicted'
+  }
+  if (evidence.expect && !output.includes(evidence.expect)) {
+    return output === '' ? 'silent-check' : 'contradicted'
+  }
+  return 'verified'
+}

src/index.ts

@@ -4,6 +4,7 @@ export * from './agent-candidate'
 export * from './benchmarks/index'
 export * from './changes'
 export * from './chunking'
+export * from './claim-evidence'
 export * from './claim-grounding'
 export * from './claim-ledger'
 export * from './collection-research-driver'
@@ -48,6 +49,7 @@ export * from './research-driving-driver'
 export * from './research-loop'
 export * from './retrieval-eval'
 export * from './retrieval-optimization'
+export * from './run-scoped'
 export * from './schemas'
 export * from './search'
 export * from './sources'

src/run-scoped.test.ts

@@ -0,0 +1,76 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { createRunScopedStores } from './run-scoped'
+
+let root: string
+let shared: string
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), 'run-scoped-'))
+  shared = await mkdtemp(join(tmpdir(), 'run-shared-'))
+})
+afterEach(async () => {
+  await rm(root, { recursive: true, force: true })
+  await rm(shared, { recursive: true, force: true })
+})
+
+async function addPage(storeRoot: string, name: string, body: string): Promise<void> {
+  await writeFile(join(storeRoot, 'knowledge', name), `---\ntitle: ${name}\n---\n\n${body}\n`)
+}
+
+describe('createRunScopedStores', () => {
+  it('isolates writes per run and labels chain reads by origin', async () => {
+    const stores = createRunScopedStores({ root, sharedRoot: shared })
+    const a = await stores.init('run-a')
+    await addPage(join(root, 'run-a', 'knowledge-base'), 'a-finding.md', 'learned by a')
+    await stores.init('run-b', { parentRunId: 'run-a' })
+    await addPage(join(root, 'run-b', 'knowledge-base'), 'b-finding.md', 'learned by b')
+    expect(a.root).toContain('run-a')
+
+    const chain = await stores.loadChain('run-b')
+    const origins = new Map(chain.map((entry) => [entry.page.title, entry.origin]))
+    expect(origins.get('b-finding.md')).toBe('here')
+    expect(origins.get('a-finding.md')).toBe('inherited:run-a')
+  })
+
+  it('a fresh run inherits nothing from siblings — only declared ancestry reaches it', async () => {
+    const stores = createRunScopedStores({ root })
+    await stores.init('arm-1')
+    await addPage(join(root, 'arm-1', 'knowledge-base'), 'arm1-claim.md', 'arm 1 believes this')
+    await stores.init('arm-2')
+
+    const chain = await stores.loadChain('arm-2')
+    expect(chain.map((entry) => entry.page.title)).not.toContain('arm1-claim.md')
+  })
+
+  it('reads the shared store last, labeled shared, without any run writing it', async () => {
+    const stores = createRunScopedStores({ root, sharedRoot: join(shared, 'kb') })
+    const sharedStores = createRunScopedStores({
+      root: shared,
+      runStorePath: () => join(shared, 'kb'),
+    })
+    await sharedStores.init('shared')
+    await addPage(join(shared, 'kb'), 'lab-lesson.md', 'curated instrument knowledge')
+    await stores.init('run-x')
+
+    const chain = await stores.loadChain('run-x')
+    const lesson = chain.find((entry) => entry.page.title === 'lab-lesson.md')
+    expect(lesson?.origin).toBe('shared')
+  })
+
+  it('ends the chain at a run with no lineage record instead of erroring', async () => {
+    const stores = createRunScopedStores({ root })
+    await stores.init('child', { parentRunId: 'never-initialized-parent' })
+    await expect(stores.lineage('child')).resolves.toEqual(['never-initialized-parent'])
+    await expect(stores.loadChain('child')).resolves.toEqual([])
+  })
+
+  it('refuses a lineage cycle loudly', async () => {
+    const stores = createRunScopedStores({ root })
+    await stores.init('a', { parentRunId: 'b' })
+    await stores.init('b', { parentRunId: 'a' })
+    await expect(stores.lineage('a')).rejects.toThrow(/cycle/)
+  })
+})

src/run-scoped.ts

@@ -0,0 +1,130 @@
+/**
+ * Run-scoped knowledge stores with lineage-chain reads.
+ *
+ * One store per run, physically isolated, with inheritance only by declared ancestry. The design
+ * answers a failure mode a discovery campaign paid for twice: when every run writes one shared
+ * store, a false "measured" claim from one arm becomes the next arm's settled provenance — twice
+ * in that campaign a wrong claim crossed run boundaries and became standing instruction before
+ * anyone re-derived it. Isolation makes "what did THIS run learn" a directory listing, and the
+ * knowledge delta between two runs a diff of two directories.
+ *
+ * A branch still inherits: reads walk the run's own store, then each ancestor up a recorded
+ * parent chain, then an optional shared store — every page labeled with where it came from,
+ * because "I established this", "an earlier attempt believed this", and "the lab curates this"
+ * are three different provenance claims and a reader must be able to weigh them differently.
+ */
+import { join } from 'node:path'
+import { isMissingFile, readRegularFileWithinRoot } from './durable-fs'
+import type { KnowledgeLayout } from './store'
+import { initKnowledgeBase, loadKnowledgePages, writeJson } from './store'
+import type { KnowledgePage } from './types'
+
+/** Where a page in a chained read came from. */
+export type PageOrigin = 'here' | `inherited:${string}` | 'shared'
+
+export interface OriginatedPage {
+  page: KnowledgePage
+  origin: PageOrigin
+}
+
+export interface RunLineageRecord {
+  runId: string
+  parentRunId: string | null
+  createdAt: string
+}
+
+export const RUN_LINEAGE_BASENAME = 'lineage.json'
+
+export interface RunScopedStoresOptions {
+  /** The root under which per-run stores live. */
+  root: string
+  /** Store path for a run; defaults to `<root>/<runId>/knowledge-base`. */
+  runStorePath?: (runId: string) => string
+  /**
+   * The curated store every run may read but no run writes — promotion into it is a deliberate
+   * act outside this module. Searched last, labeled `shared`.
+   */
+  sharedRoot?: string
+}
+
+/** Ancestry chains longer than this indicate a cycle or a runaway, not a real lineage. */
+const MAX_LINEAGE_HOPS = 64
+
+export interface RunScopedStores {
+  /** Create (or open) a run's store, recording its parent at creation time. */
+  init(runId: string, options?: { parentRunId?: string | null }): Promise<KnowledgeLayout>
+  /** The ancestor chain of a run, nearest first, resolved from records written at init. */
+  lineage(runId: string): Promise<string[]>
+  /**
+   * Every page visible to a run: its own, then each ancestor's, then the shared store's, each
+   * labeled with its origin. Later stores never shadow earlier ones — a reader sees both copies
+   * of a twice-recorded claim and the labels that distinguish them.
+   */
+  loadChain(runId: string): Promise<OriginatedPage[]>
+}
+
+export function createRunScopedStores(options: RunScopedStoresOptions): RunScopedStores {
+  const storePath =
+    options.runStorePath ?? ((runId: string) => join(options.root, runId, 'knowledge-base'))
+
+  async function parentOf(runId: string): Promise<string | null> {
+    try {
+      const snapshot = await readRegularFileWithinRoot(storePath(runId), RUN_LINEAGE_BASENAME)
+      return (JSON.parse(snapshot.bytes.toString('utf8')) as RunLineageRecord).parentRunId
+    } catch (error) {
+      // A run created before lineage recording (or outside it) simply ends the chain: absent
+      // ancestry is absent, not an error — the chain is only as deep as what was declared.
+      if (isMissingFile(error)) return null
+      throw error
+    }
+  }
+
+  return {
+    async init(runId, initOptions = {}) {
+      const layout = await initKnowledgeBase(storePath(runId))
+      const record: RunLineageRecord = {
+        runId,
+        parentRunId: initOptions.parentRunId ?? null,
+        createdAt: new Date().toISOString(),
+      }
+      await writeJson(join(storePath(runId), RUN_LINEAGE_BASENAME), record)
+      return layout
+    },
+
+    async lineage(runId) {
+      const chain: string[] = []
+      const seen = new Set<string>([runId])
+      let current: string | null = runId
+      for (let hop = 0; hop < MAX_LINEAGE_HOPS && current !== null; hop += 1) {
+        current = await parentOf(current)
+        if (current === null) break
+        if (seen.has(current)) {
+          throw new Error(`run lineage cycle: ${current} is its own ancestor (via ${runId})`)
+        }
+        seen.add(current)
+        chain.push(current)
+      }
+      return chain
+    },
+
+    async loadChain(runId) {
+      const out: OriginatedPage[] = []
+      const readInto = async (root: string, origin: PageOrigin) => {
+        let pages: KnowledgePage[]
+        try {
+          pages = await loadKnowledgePages(root)
+        } catch (error) {
+          if (isMissingFile(error)) return
+          throw error
+        }
+        for (const page of pages) out.push({ page, origin })
+      }
+      await readInto(storePath(runId), 'here')
+      for (const ancestor of await this.lineage(runId)) {
+        await readInto(storePath(ancestor), `inherited:${ancestor}`)
+      }
+      if (options.sharedRoot) await readInto(options.sharedRoot, 'shared')
+      return out
+    },
+  }
+}