diff --git a/src/claim-evidence.test.ts b/src/claim-evidence.test.ts new file mode 100644 index 0000000..42991ec --- /dev/null +++ b/src/claim-evidence.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import { assertGradeableEvidence, UncheckableClaimError, verdictFor } from './claim-evidence' + +describe('assertGradeableEvidence', () => { + it('refuses rung 4+ without a check — a self-grade must fail at record time', () => { + expect(() => assertGradeableEvidence({ rung: 4 })).toThrow(UncheckableClaimError) + expect(() => assertGradeableEvidence({ rung: 5 })).toThrow(UncheckableClaimError) + }) + it('passes rung 4+ with a check, and any rung below the threshold', () => { + expect(assertGradeableEvidence({ rung: 4, check: 'true' }).rung).toBe(4) + expect(assertGradeableEvidence({ rung: 3 }).rung).toBe(3) + }) +}) + +describe('verdictFor — the calibration cases that were graded wrong before this lattice', () => { + it('a silent assert is not a contradiction', () => { + // exit 0, prints nothing: the expectation can never appear, and grading this 'contradicted' + // once flunked three TRUE claims on first contact with real data. + expect( + verdictFor({ rung: 4, check: 'x', expect: 'True' }, { exitCode: 0, stdout: '', stderr: '' }), + ).toBe('silent-check') + }) + it('a printed decisive value verifies', () => { + expect( + verdictFor( + { rung: 4, check: 'x', expect: '2983' }, + { exitCode: 0, stdout: 'roots_checked=2983', stderr: '' }, + ), + ).toBe('verified') + }) + it('a missing input blames the environment, not the claim', () => { + expect( + verdictFor( + { rung: 4, check: 'x' }, + { exitCode: 1, stdout: '', stderr: 'FileNotFoundError: k3.json' }, + ), + ).toBe('unrunnable') + }) + it('a passing command printing the wrong value is a real contradiction', () => { + expect( + verdictFor( + { rung: 4, check: 'x', expect: '7' }, + { exitCode: 0, stdout: 'value=8', stderr: '' }, + ), + ).toBe('contradicted') + }) + it('rung 4+ with no check is uncheckable regardless of execution', () => { + expect(verdictFor({ rung: 4 }, null)).toBe('uncheckable') + }) + it('a check that could not execute at all is unrunnable', () => { + expect(verdictFor({ rung: 4, check: 'x' }, null)).toBe('unrunnable') + }) +}) diff --git a/src/claim-evidence.ts b/src/claim-evidence.ts new file mode 100644 index 0000000..ff78de7 --- /dev/null +++ b/src/claim-evidence.ts @@ -0,0 +1,116 @@ +/** + * The evidence contract for claims that machines, not authors, get to grade. + * + * Grown in a discovery campaign that learned each rule by paying for it. Self-reported + * verification is a dead channel: an entire run's twenty-eight true claims scored zero because + * nothing carried a check a grader could re-execute, and a sibling lab's cells once produced six + * false certifications in seventeen deliveries without a single agent lying — the format betrayed + * them. The countermeasure is mechanical: a claim above a threshold must carry the command that + * re-establishes it, and an independent grader re-runs that command blind. + * + * The verdict lattice below is calibrated, not designed: its first contact with real claims + * graded three TRUE results as refuted, because a silent assert exits zero printing nothing and + * the expectation could never match empty output. Every distinction here exists because collapsing + * it blamed the wrong party — a claim for its environment, or an author for their formatting. + */ + +/** + * How far up the ladder a claim's evidence actually reaches. The rungs are five different + * statements and only the top two are verification: + * + * 1 — it parses / exists 3 — its tests pass 5 — independently re-derived + * 2 — it imports / loads 4 — reproduces a claimed value + * + * Reporting a low rung in the vocabulary of a high one is the most expensive error available to a + * knowledge system: it is indistinguishable from success and propagates as settled provenance. + */ +export type EvidenceRung = 1 | 2 | 3 | 4 | 5 + +/** The rung at and above which a claim must be machine-checkable to be recorded at that rung. */ +export const CHECKABLE_RUNG_THRESHOLD = 4 + +export interface ClaimEvidence { + rung: EvidenceRung + /** + * A shell command that re-establishes the claim when run by a grader the author cannot edit, + * from a working directory the author must not assume. Required at rung 4 and above. + */ + check?: string + /** + * A substring the check's stdout must contain — the decisive value, printed. A check that + * exits zero but prints nothing cannot confirm a value; authors should print the number. + */ + expect?: string + /** Where the artifact backing the claim lives, for humans following the trail. */ + evidencePath?: string +} + +export class UncheckableClaimError extends Error { + constructor(rung: EvidenceRung) { + super( + `a claim at rung ${rung} asserts "a command reproduces a value" and must carry that ` + + `command (evidence.check); without one it is a self-grade, which is rung 3 at most`, + ) + this.name = 'UncheckableClaimError' + } +} + +/** + * Refuse the evidence shapes that made self-grading possible. Call at record time, not grade + * time: by grading it is too late — the ungradeable claim has already circulated as verified. + */ +export function assertGradeableEvidence(evidence: ClaimEvidence): ClaimEvidence { + if (evidence.rung >= CHECKABLE_RUNG_THRESHOLD && !evidence.check) { + throw new UncheckableClaimError(evidence.rung) + } + return evidence +} + +/** + * What a grader may conclude from re-executing a claim's check. + * + * verified exit 0, and the expectation (if any) appears in output + * silent-check exit 0, expectation given, output empty — the check passed but proves nothing + * about the expected value; the author should make the check PRINT it + * contradicted the check ran and refuted the claim: nonzero exit, or non-empty output that + * lacks the expectation + * unrunnable the check itself could not execute (missing input, missing module) — an + * environment verdict, never a claim verdict + * uncheckable rung demanded a check and none was recorded — a self-grade, counted against + */ +export type ClaimVerdict = + | 'verified' + | 'silent-check' + | 'contradicted' + | 'unrunnable' + | 'uncheckable' + +/** The error signatures that mean the check could not run, as opposed to ran and failed. */ +const UNRUNNABLE_SIGNATURES = + /No such file|FileNotFoundError|ModuleNotFoundError|command not found|ENOENT/ + +export interface CheckExecution { + exitCode: number + stdout: string + stderr: string +} + +/** + * The calibrated verdict function, pure so every grader shares one semantics. Callers execute the + * check however their environment requires and pass the observation; this function only judges. + */ +export function verdictFor( + evidence: Pick, + execution: CheckExecution | null, +): ClaimVerdict { + if (evidence.rung >= CHECKABLE_RUNG_THRESHOLD && !evidence.check) return 'uncheckable' + if (!execution) return 'unrunnable' + const output = `${execution.stdout}\n${execution.stderr}`.trim() + if (execution.exitCode !== 0) { + return UNRUNNABLE_SIGNATURES.test(output) ? 'unrunnable' : 'contradicted' + } + if (evidence.expect && !output.includes(evidence.expect)) { + return output === '' ? 'silent-check' : 'contradicted' + } + return 'verified' +} diff --git a/src/index.ts b/src/index.ts index 962c0a4..5340858 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ export * from './agent-candidate' export * from './benchmarks/index' export * from './changes' export * from './chunking' +export * from './claim-evidence' export * from './claim-grounding' export * from './claim-ledger' export * from './collection-research-driver' @@ -48,6 +49,7 @@ export * from './research-driving-driver' export * from './research-loop' export * from './retrieval-eval' export * from './retrieval-optimization' +export * from './run-scoped' export * from './schemas' export * from './search' export * from './sources' diff --git a/src/run-scoped.test.ts b/src/run-scoped.test.ts new file mode 100644 index 0000000..88e46c3 --- /dev/null +++ b/src/run-scoped.test.ts @@ -0,0 +1,76 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { createRunScopedStores } from './run-scoped' + +let root: string +let shared: string + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), 'run-scoped-')) + shared = await mkdtemp(join(tmpdir(), 'run-shared-')) +}) +afterEach(async () => { + await rm(root, { recursive: true, force: true }) + await rm(shared, { recursive: true, force: true }) +}) + +async function addPage(storeRoot: string, name: string, body: string): Promise { + await writeFile(join(storeRoot, 'knowledge', name), `---\ntitle: ${name}\n---\n\n${body}\n`) +} + +describe('createRunScopedStores', () => { + it('isolates writes per run and labels chain reads by origin', async () => { + const stores = createRunScopedStores({ root, sharedRoot: shared }) + const a = await stores.init('run-a') + await addPage(join(root, 'run-a', 'knowledge-base'), 'a-finding.md', 'learned by a') + await stores.init('run-b', { parentRunId: 'run-a' }) + await addPage(join(root, 'run-b', 'knowledge-base'), 'b-finding.md', 'learned by b') + expect(a.root).toContain('run-a') + + const chain = await stores.loadChain('run-b') + const origins = new Map(chain.map((entry) => [entry.page.title, entry.origin])) + expect(origins.get('b-finding.md')).toBe('here') + expect(origins.get('a-finding.md')).toBe('inherited:run-a') + }) + + it('a fresh run inherits nothing from siblings — only declared ancestry reaches it', async () => { + const stores = createRunScopedStores({ root }) + await stores.init('arm-1') + await addPage(join(root, 'arm-1', 'knowledge-base'), 'arm1-claim.md', 'arm 1 believes this') + await stores.init('arm-2') + + const chain = await stores.loadChain('arm-2') + expect(chain.map((entry) => entry.page.title)).not.toContain('arm1-claim.md') + }) + + it('reads the shared store last, labeled shared, without any run writing it', async () => { + const stores = createRunScopedStores({ root, sharedRoot: join(shared, 'kb') }) + const sharedStores = createRunScopedStores({ + root: shared, + runStorePath: () => join(shared, 'kb'), + }) + await sharedStores.init('shared') + await addPage(join(shared, 'kb'), 'lab-lesson.md', 'curated instrument knowledge') + await stores.init('run-x') + + const chain = await stores.loadChain('run-x') + const lesson = chain.find((entry) => entry.page.title === 'lab-lesson.md') + expect(lesson?.origin).toBe('shared') + }) + + it('ends the chain at a run with no lineage record instead of erroring', async () => { + const stores = createRunScopedStores({ root }) + await stores.init('child', { parentRunId: 'never-initialized-parent' }) + await expect(stores.lineage('child')).resolves.toEqual(['never-initialized-parent']) + await expect(stores.loadChain('child')).resolves.toEqual([]) + }) + + it('refuses a lineage cycle loudly', async () => { + const stores = createRunScopedStores({ root }) + await stores.init('a', { parentRunId: 'b' }) + await stores.init('b', { parentRunId: 'a' }) + await expect(stores.lineage('a')).rejects.toThrow(/cycle/) + }) +}) diff --git a/src/run-scoped.ts b/src/run-scoped.ts new file mode 100644 index 0000000..f6b0e2d --- /dev/null +++ b/src/run-scoped.ts @@ -0,0 +1,130 @@ +/** + * Run-scoped knowledge stores with lineage-chain reads. + * + * One store per run, physically isolated, with inheritance only by declared ancestry. The design + * answers a failure mode a discovery campaign paid for twice: when every run writes one shared + * store, a false "measured" claim from one arm becomes the next arm's settled provenance — twice + * in that campaign a wrong claim crossed run boundaries and became standing instruction before + * anyone re-derived it. Isolation makes "what did THIS run learn" a directory listing, and the + * knowledge delta between two runs a diff of two directories. + * + * A branch still inherits: reads walk the run's own store, then each ancestor up a recorded + * parent chain, then an optional shared store — every page labeled with where it came from, + * because "I established this", "an earlier attempt believed this", and "the lab curates this" + * are three different provenance claims and a reader must be able to weigh them differently. + */ +import { join } from 'node:path' +import { isMissingFile, readRegularFileWithinRoot } from './durable-fs' +import type { KnowledgeLayout } from './store' +import { initKnowledgeBase, loadKnowledgePages, writeJson } from './store' +import type { KnowledgePage } from './types' + +/** Where a page in a chained read came from. */ +export type PageOrigin = 'here' | `inherited:${string}` | 'shared' + +export interface OriginatedPage { + page: KnowledgePage + origin: PageOrigin +} + +export interface RunLineageRecord { + runId: string + parentRunId: string | null + createdAt: string +} + +export const RUN_LINEAGE_BASENAME = 'lineage.json' + +export interface RunScopedStoresOptions { + /** The root under which per-run stores live. */ + root: string + /** Store path for a run; defaults to `//knowledge-base`. */ + runStorePath?: (runId: string) => string + /** + * The curated store every run may read but no run writes — promotion into it is a deliberate + * act outside this module. Searched last, labeled `shared`. + */ + sharedRoot?: string +} + +/** Ancestry chains longer than this indicate a cycle or a runaway, not a real lineage. */ +const MAX_LINEAGE_HOPS = 64 + +export interface RunScopedStores { + /** Create (or open) a run's store, recording its parent at creation time. */ + init(runId: string, options?: { parentRunId?: string | null }): Promise + /** The ancestor chain of a run, nearest first, resolved from records written at init. */ + lineage(runId: string): Promise + /** + * Every page visible to a run: its own, then each ancestor's, then the shared store's, each + * labeled with its origin. Later stores never shadow earlier ones — a reader sees both copies + * of a twice-recorded claim and the labels that distinguish them. + */ + loadChain(runId: string): Promise +} + +export function createRunScopedStores(options: RunScopedStoresOptions): RunScopedStores { + const storePath = + options.runStorePath ?? ((runId: string) => join(options.root, runId, 'knowledge-base')) + + async function parentOf(runId: string): Promise { + try { + const snapshot = await readRegularFileWithinRoot(storePath(runId), RUN_LINEAGE_BASENAME) + return (JSON.parse(snapshot.bytes.toString('utf8')) as RunLineageRecord).parentRunId + } catch (error) { + // A run created before lineage recording (or outside it) simply ends the chain: absent + // ancestry is absent, not an error — the chain is only as deep as what was declared. + if (isMissingFile(error)) return null + throw error + } + } + + return { + async init(runId, initOptions = {}) { + const layout = await initKnowledgeBase(storePath(runId)) + const record: RunLineageRecord = { + runId, + parentRunId: initOptions.parentRunId ?? null, + createdAt: new Date().toISOString(), + } + await writeJson(join(storePath(runId), RUN_LINEAGE_BASENAME), record) + return layout + }, + + async lineage(runId) { + const chain: string[] = [] + const seen = new Set([runId]) + let current: string | null = runId + for (let hop = 0; hop < MAX_LINEAGE_HOPS && current !== null; hop += 1) { + current = await parentOf(current) + if (current === null) break + if (seen.has(current)) { + throw new Error(`run lineage cycle: ${current} is its own ancestor (via ${runId})`) + } + seen.add(current) + chain.push(current) + } + return chain + }, + + async loadChain(runId) { + const out: OriginatedPage[] = [] + const readInto = async (root: string, origin: PageOrigin) => { + let pages: KnowledgePage[] + try { + pages = await loadKnowledgePages(root) + } catch (error) { + if (isMissingFile(error)) return + throw error + } + for (const page of pages) out.push({ page, origin }) + } + await readInto(storePath(runId), 'here') + for (const ancestor of await this.lineage(runId)) { + await readInto(storePath(ancestor), `inherited:${ancestor}`) + } + if (options.sharedRoot) await readInto(options.sharedRoot, 'shared') + return out + }, + } +}