diff --git a/CHANGELOG.md b/CHANGELOG.md index bcfa68da..13ffafa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,40 @@ # Changelog +## Unreleased + +### Bug Fixes + +- add `titles` column migration for `person_observations` — existing + databases predate the `title` → `titles` rename and CREATE TABLE IF NOT + EXISTS is a no-op on already-created tables; the first write hits `table + has no column named titles`. Wired before `PersonObservationRepo.setupDatabase()`. +- add `loi_date` column migration for `spac`, `spac_deal`, `spac_history` — + same CREATE TABLE IF NOT EXISTS problem after the LOI lifecycle stage + landed. `spac.status = "loi"` needs no DDL (`TypeStringEnum` emits plain + TEXT with no CHECK constraint). +- bump the person resolver to `2.0.0` — `PersonNormalization` now strips + periods/commas and folds typographic apostrophes, which changes the + `(normalized_first, normalized_middle, normalized_last, normalized_suffix)` + tuple `PersonResolver.personKey` looks up by. Post-fold observations under + the same `resolver_version` silently miss pre-fold canonicals and mint + duplicates. + +### Operator Ceremony (person resolver 2.0.0) + +Existing DBs must run through the version-bump ceremony to migrate +identities across the fold. Aliases must be re-applied manually against the +fresh 2.0.0 canonical UUIDs (`sec canonical person alias-list` before the +ceremony; re-issue `sec canonical person alias` after). + +```sh +sec version start-dev resolver person 2.0.0 --bump major \ + --notes "PersonNormalization punctuation/typographic fold changes key tuple" +sec resolve --kind person --resolver-version 2.0.0 --all +sec version coverage resolver person +sec version promote resolver person +sec version drop-previous resolver person +``` + ## 0.0.9 ### Chores diff --git a/CLAUDE.md b/CLAUDE.md index bdfdd17c..0abc4d9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,6 +41,28 @@ sec version drop-previous resolver company sec version drop-previous extractor ``` +#### Resolver version bumps + +When the resolver's normalized key tuple changes (e.g. `PersonNormalization` +now strips periods/commas and folds typographic apostrophes → the +`normalized_first/middle/last/suffix` tuple `PersonResolver.personKey` looks up +by), bump the resolver's semver so post-change lookups do not silently miss +pre-change canonicals and mint duplicates under the same `resolver_version`. +Worked example for the person resolver 1.0.0 → 2.0.0: + +```bash +sec version start-dev resolver person 2.0.0 --bump major \ + --notes "PersonNormalization punctuation/typographic fold changes key tuple" +sec resolve --kind person --resolver-version 2.0.0 --all +sec version coverage resolver person +sec version promote resolver person +sec version drop-previous resolver person +``` + +Aliases must be re-applied manually against the fresh 2.0.0 canonical UUIDs +(`sec canonical person alias-list` before the ceremony; re-issue +`sec canonical person alias` after). + ### S-1 extraction S-1 prospectuses are narrative HTML (not structured XML), so the `S-1` extractor diff --git a/src/cli/groups/version.test.ts b/src/cli/groups/version.test.ts index ba8839a3..d78c1109 100644 --- a/src/cli/groups/version.test.ts +++ b/src/cli/groups/version.test.ts @@ -283,6 +283,34 @@ describe("sec version CLI", () => { } }); + it("start-dev resolver person 2.0.0 --bump major snapshots observation count", async () => { + const dir = mkdtempSync(join(tmpdir(), "sec-version-test-")); + try { + const setup = await runCli(["db", "setup"], dir); + expect(setup.exitCode).toBe(0); + + const result = await runCli( + ["version", "start-dev", "resolver", "person", "2.0.0", "--bump", "major"], + dir + ); + expect(result.exitCode).toBe(0); + // Must NOT be refused as "not yet supported". + expect(result.stderr + result.stdout).not.toMatch(/not yet supported/); + + // No observations were seeded, so target_count === PersonObservationRepo.count() === 0. + const coverage = await runCli( + ["version", "coverage", "resolver", "person", "--format", "json"], + dir + ); + expect(coverage.exitCode).toBe(0); + const parsed = JSON.parse(coverage.stdout); + // computeResolverCoverage returns denominator (observation count). + expect(parsed.denominator).toBe(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 15000); + it("rejects start-dev for an unknown extractor id", async () => { const dir = mkdtempSync(join(tmpdir(), "sec-version-test-")); try { diff --git a/src/cli/groups/version.ts b/src/cli/groups/version.ts index 170f4762..1dd8b1b1 100644 --- a/src/cli/groups/version.ts +++ b/src/cli/groups/version.ts @@ -6,8 +6,10 @@ import type { Command } from "commander"; import { globalServiceRegistry } from "workglow"; -import type { ResolverId } from "../../resolver/resolverIds"; +import { isFamilyResolverId, type ResolverId } from "../../resolver/resolverIds"; import { FILING_REPOSITORY_TOKEN } from "../../storage/filing/FilingSchema"; +import { CompanyObservationRepo } from "../../storage/observation/CompanyObservationRepo"; +import { PersonObservationRepo } from "../../storage/observation/PersonObservationRepo"; import { dropNext as dropNextCeremony, dropPrevious as dropPreviousCeremony, @@ -61,20 +63,31 @@ function assertFormat(s: string): asserts s is "table" | "json" { } /** - * For a major-bump extractor start-dev, snapshot the count of filings - * handled by this extractor. Stored on the next-slot row; used as the - * promote-gate denominator. + * For a major-bump start-dev, snapshot the count of items the ceremony's + * coverage gate will be denominated over. Stored on the next-slot row. * - * This is extractor-specific today. When resolvers gain a major - * dev-cycle, they will need their own kind-aware snapshot strategy - * (count of observations? count of canonical identities? TBD). Add a - * dispatch table keyed on ComponentKind when resolver support lands. + * Extractor snapshots count the filings the extractor handles (via + * `FORM_TO_EXTRACTOR_ID` → filing forms). Resolver snapshots count the + * observations the resolver ranges over (person / company). Family-tier + * resolvers have no observation model and are refused, matching the + * behavior of `computeResolverCoverage`. */ async function snapshotTargetCount(kind: ComponentKind, id: string): Promise { - if (kind !== "extractor") { - throw new Error( - `snapshotTargetCount: kind '${kind}' is not yet supported; only 'extractor' has a snapshot strategy. Add resolver support when implemented.` - ); + if (kind === "resolver") { + if (isFamilyResolverId(id)) { + throw new Error( + `Coverage snapshots not supported for family-tier resolvers (got '${id}'). ` + + `Family resolvers track membership, not observation identity-links.` + ); + } + const resolverId = id as ResolverId; + if (resolverId === "person") { + return await new PersonObservationRepo().count(); + } + if (resolverId === "company") { + return await new CompanyObservationRepo().count(); + } + throw new Error(`snapshotTargetCount: unknown resolver id '${id}'`); } const filingRepo = globalServiceRegistry.get(FILING_REPOSITORY_TOKEN); const forms = Object.entries(FORM_TO_EXTRACTOR_ID) diff --git a/src/config/setupAllDatabases.ts b/src/config/setupAllDatabases.ts index e05060af..7d5de127 100644 --- a/src/config/setupAllDatabases.ts +++ b/src/config/setupAllDatabases.ts @@ -54,6 +54,7 @@ import { SPAC_REPOSITORY_TOKEN } from "../storage/spac/SpacSchema"; import { SPAC_DEAL_REPOSITORY_TOKEN } from "../storage/spac/SpacDealSchema"; import { SPAC_EVENT_REPOSITORY_TOKEN } from "../storage/spac/SpacEventSchema"; import { SPAC_HISTORY_REPOSITORY_TOKEN } from "../storage/spac/SpacHistorySchema"; +import { migrateSpacLoiColumns } from "../storage/spac/SpacLoiColumnsMigration"; import { SPAC_MERGER_EXTRACTION_REPOSITORY_TOKEN } from "../storage/spac/SpacMergerExtractionSchema"; import { SPAC_REDEMPTION_EXTRACTION_REPOSITORY_TOKEN } from "../storage/spac/SpacRedemptionExtractionSchema"; import { SPAC_LOI_EXTRACTION_REPOSITORY_TOKEN } from "../storage/spac/SpacLoiExtractionSchema"; @@ -90,6 +91,7 @@ import { PERSON_IDENTITY_LINK_REPOSITORY_TOKEN } from "../storage/canonical/Pers import { CURRENT_CANONICAL_VIEW_DDL } from "../storage/canonical/views"; import { COMPANY_OBSERVATION_REPOSITORY_TOKEN } from "../storage/observation/CompanyObservationSchema"; import { PERSON_OBSERVATION_REPOSITORY_TOKEN } from "../storage/observation/PersonObservationSchema"; +import { migratePersonObservationTitles } from "../storage/observation/PersonObservationTitlesMigration"; import { OBSERVATION_PROVENANCE_REPOSITORY_TOKEN } from "../storage/provenance/ObservationProvenanceSchema"; import { BENEFICIAL_OWNERSHIP_REPOSITORY_TOKEN } from "../storage/beneficial-ownership/BeneficialOwnershipSchema"; import { RELATED_PARTY_TRANSACTION_REPOSITORY_TOKEN } from "../storage/related-party/RelatedPartyTransactionSchema"; @@ -146,6 +148,11 @@ export async function setupAllDatabases(): Promise { await globalServiceRegistry.get(REGA_SERVICE_PROVIDER_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(REGA_FINANCIAL_DATA_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(REGA_EQUITY_CLASS_REPOSITORY_TOKEN).setupDatabase(); + // Add loi_date to the SPAC tables before setupDatabase() creates the current + // shape; CREATE TABLE IF NOT EXISTS is a no-op on an existing table, so an + // upgraded DB never gains loi_date on its own. The status enum's new "loi" + // value needs no DDL — TypeStringEnum emits plain TEXT with no CHECK. + await migrateSpacLoiColumns(); await globalServiceRegistry.get(SPAC_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(SPAC_DEAL_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(SPAC_EVENT_REPOSITORY_TOKEN).setupDatabase(); @@ -160,6 +167,10 @@ export async function setupAllDatabases(): Promise { await globalServiceRegistry.get(COMPONENT_VERSION_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(EXTRACTOR_RUN_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(VERSION_EVENT_REPOSITORY_TOKEN).setupDatabase(); + // Migrate legacy `title TEXT` → `titles TEXT` before setupDatabase() creates + // the current shape; CREATE TABLE IF NOT EXISTS is a no-op on an existing + // table, so an upgraded DB never gains the renamed column on its own. + await migratePersonObservationTitles(); await globalServiceRegistry.get(PERSON_OBSERVATION_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(COMPANY_OBSERVATION_REPOSITORY_TOKEN).setupDatabase(); await globalServiceRegistry.get(OBSERVATION_PROVENANCE_REPOSITORY_TOKEN).setupDatabase(); diff --git a/src/resolver/PersonResolver.test.ts b/src/resolver/PersonResolver.test.ts index 79d35a19..5d02b4a9 100644 --- a/src/resolver/PersonResolver.test.ts +++ b/src/resolver/PersonResolver.test.ts @@ -161,6 +161,74 @@ describe("PersonResolver.resolve", () => { expect(result).toBe("alias-target"); }); + it("cross-version isolation: 1.0.0 canonical is not returned under 2.0.0 lookup", async () => { + // Seed a canonical person from the pre-fold tuple (a "1.0.0" resolver + // would have written this). Now resolve an observation whose normalized + // tuple is the post-fold shape under a fresh 2.0.0 resolver — the + // resolver_version scope keeps the 1.0.0 row out of scope and mints a + // new canonical id. + const preFoldRow: CanonicalPerson = { + canonical_person_id: "canon-1.0.0", + resolver_version: "1.0.0", + display_first: "Richard", + display_middle: "J.", + display_last: "Boyle", + display_suffix: "Jr.", + cik: null, + normalized_first: "richard", + normalized_middle: "j.", + normalized_last: "boyle", + normalized_suffix: "jr.", + source_filing_issuer_cik: 100, + created_at: "2026-05-22T00:00:00.000Z", + }; + await setup.canonRepo.create(preFoldRow); + + const v2Resolver = new PersonResolver({ + canonicalPersonRepo: setup.canonRepo, + canonicalPersonAliasRepo: setup.aliasRepo, + activeResolverVersion: "2.0.0", + }); + const v2Id = await v2Resolver.resolve( + obs({ + normalized_first: "richard", + normalized_middle: "j", + normalized_last: "boyle", + normalized_suffix: "jr", + source_filing_issuer_cik: 100, + }) + ); + expect(v2Id).not.toBe("canon-1.0.0"); + }); + + it("2.0.0 idempotency: two equal post-fold keys return the same UUID", async () => { + const v2Resolver = new PersonResolver({ + canonicalPersonRepo: setup.canonRepo, + canonicalPersonAliasRepo: setup.aliasRepo, + activeResolverVersion: "2.0.0", + }); + const a = await v2Resolver.resolve( + obs({ + normalized_first: "richard", + normalized_middle: "j", + normalized_last: "boyle", + normalized_suffix: "jr", + source_filing_issuer_cik: 100, + }) + ); + const b = await v2Resolver.resolve( + obs({ + normalized_first: "richard", + normalized_middle: "j", + normalized_last: "boyle", + normalized_suffix: "jr", + source_filing_issuer_cik: 100, + observation_id: 2, + }) + ); + expect(a).toBe(b); + }); + it("serialises alias lookup inside the per-key mutex (no overlapping alias.resolve calls)", async () => { // The previous regression test stubbed `aliasRepo.resolve` to always // return a constant id regardless of input, so both pre-fix (lookup diff --git a/src/storage/observation/PersonObservationTitlesMigration.test.ts b/src/storage/observation/PersonObservationTitlesMigration.test.ts new file mode 100644 index 00000000..967ebb0c --- /dev/null +++ b/src/storage/observation/PersonObservationTitlesMigration.test.ts @@ -0,0 +1,211 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { mkdtempSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { globalServiceRegistry, Sqlite } from "workglow"; +import { resetDependencyInjectionsForTesting } from "../../config/TestingDI"; +import { setupAllDatabases } from "../../config/setupAllDatabases"; +import { SEC_DB_FOLDER, SEC_DB_NAME, SEC_DB_TYPE } from "../../config/tokens"; +import { closeDb, getDb } from "../../util/db"; +import { migratePersonObservationTitles } from "./PersonObservationTitlesMigration"; +import { PersonObservationRepo } from "./PersonObservationRepo"; + +const TEST_DB_NAME = "person_obs_titles_migration_test"; + +describe("migratePersonObservationTitles (sqlite)", () => { + let tmpDir: string; + + beforeEach(async () => { + resetDependencyInjectionsForTesting(); + closeDb(); + if (typeof Sqlite.init === "function") { + await Sqlite.init(); + } + tmpDir = mkdtempSync(join(tmpdir(), "sec-person-obs-titles-migration-")); + globalServiceRegistry.registerInstance(SEC_DB_TYPE, "sqlite"); + globalServiceRegistry.registerInstance(SEC_DB_FOLDER, tmpDir); + globalServiceRegistry.registerInstance(SEC_DB_NAME, TEST_DB_NAME); + }); + + afterEach(() => { + closeDb(); + rmSync(tmpDir, { recursive: true, force: true }); + // ServiceRegistry has no unregister API. Pin the tokens this test set to + // sentinels that fail both `dbType === "sqlite"` and `dbType === "postgres"` + // dispatch branches downstream so the in-memory test backend stays in + // charge for any test that runs after us in the same Bun process. + globalServiceRegistry.registerInstance( + SEC_DB_TYPE, + "memory" as unknown as "sqlite" | "postgres" + ); + resetDependencyInjectionsForTesting(); + }); + + it("adds titles column and backfills from legacy title", async () => { + const db = getDb(); + db.exec( + `CREATE TABLE person_observations ( + observation_id INTEGER PRIMARY KEY AUTOINCREMENT, + accession_number TEXT NOT NULL, + extractor_id TEXT NOT NULL, + extractor_version TEXT NOT NULL, + observation_index INTEGER NOT NULL, + title TEXT NULL, + created_at TEXT NOT NULL + )` + ); + const insert = db.prepare( + `INSERT INTO person_observations (accession_number, extractor_id, extractor_version, observation_index, title, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ); + insert.run("0001-25-000001", "D", "1.0.0", 0, "CEO", "2026-05-22T00:00:00.000Z"); + insert.run("0001-25-000001", "D", "1.0.0", 1, null, "2026-05-22T00:00:00.000Z"); + insert.run("0001-25-000001", "D", "1.0.0", 2, "", "2026-05-22T00:00:00.000Z"); + + await migratePersonObservationTitles(); + + const columns = db + .prepare<[], { name: string }>(`PRAGMA table_info(person_observations)`) + .all(); + expect(columns.some((c) => c.name === "titles")).toBe(true); + + const rows = db + .prepare< + [], + { observation_index: number; title: string | null; titles: string | null } + >( + `SELECT observation_index, title, titles FROM person_observations ORDER BY observation_index` + ) + .all(); + // Non-null non-empty backfills to json_array(title). + expect(rows[0].titles).not.toBeNull(); + const extracted = db + .prepare< + [], + { first_title: string | null } + >( + `SELECT JSON_EXTRACT(titles, '$[0]') as first_title FROM person_observations WHERE observation_index = 0` + ) + .get(); + expect(extracted?.first_title).toBe("CEO"); + // Null preserved. + expect(rows[1].titles).toBeNull(); + // Empty string preserved as null. + expect(rows[2].titles).toBeNull(); + }); + + it("is idempotent: running the migration twice does not re-wrap already-migrated rows", async () => { + const db = getDb(); + db.exec( + `CREATE TABLE person_observations ( + observation_id INTEGER PRIMARY KEY AUTOINCREMENT, + accession_number TEXT NOT NULL, + extractor_id TEXT NOT NULL, + extractor_version TEXT NOT NULL, + observation_index INTEGER NOT NULL, + title TEXT NULL, + created_at TEXT NOT NULL + )` + ); + db.prepare( + `INSERT INTO person_observations (accession_number, extractor_id, extractor_version, observation_index, title, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ).run("0001-25-000001", "D", "1.0.0", 0, "CEO", "2026-05-22T00:00:00.000Z"); + + await migratePersonObservationTitles(); + const firstTitles = db + .prepare<[], { titles: string | null }>(`SELECT titles FROM person_observations LIMIT 1`) + .get()?.titles; + + await migratePersonObservationTitles(); + const secondTitles = db + .prepare<[], { titles: string | null }>(`SELECT titles FROM person_observations LIMIT 1`) + .get()?.titles; + + expect(secondTitles).toBe(firstTitles); + // Sanity: single element, not doubly wrapped. + const extracted = db + .prepare< + [], + { first: string | null; second: string | null } + >( + `SELECT JSON_EXTRACT(titles, '$[0]') as first, JSON_EXTRACT(titles, '$[1]') as second FROM person_observations LIMIT 1` + ) + .get(); + expect(extracted?.first).toBe("CEO"); + expect(extracted?.second).toBeNull(); + }); + + it("is a no-op when the table does not exist yet", async () => { + // Fresh DB: no table, no error. + await migratePersonObservationTitles(); + const db = getDb(); + const remaining = db + .prepare< + [], + { name: string } + >(`SELECT name FROM sqlite_master WHERE type='table' AND name='person_observations'`) + .get(); + expect(remaining).toBeUndefined(); + }); + + it("PersonObservationRepo.upsertByNaturalKey succeeds with titles after migration", async () => { + // Start with a legacy shape (title column, no titles), run the full + // setup, then verify the repo layer can write titles arrays end to end. + const db = getDb(); + db.exec( + `CREATE TABLE person_observations ( + observation_id INTEGER PRIMARY KEY AUTOINCREMENT, + accession_number TEXT NOT NULL, + extractor_id TEXT NOT NULL, + extractor_version TEXT NOT NULL, + observation_index INTEGER NOT NULL, + source_filing_issuer_cik INTEGER NULL, + cik INTEGER NULL, + first_name TEXT NULL, + middle_name TEXT NULL, + last_name TEXT NULL, + suffix TEXT NULL, + normalized_first TEXT NULL, + normalized_middle TEXT NULL, + normalized_last TEXT NULL, + normalized_suffix TEXT NULL, + title TEXT NULL, + relationship TEXT NULL, + birth_year INTEGER NULL, + bio TEXT NULL, + raw_address_id TEXT NULL, + raw_phone_id TEXT NULL, + source_context TEXT NULL, + created_at TEXT NOT NULL, + UNIQUE (accession_number, extractor_id, observation_index) + )` + ); + + // Runs the migration then setupDatabase() on the current schema (both no-ops + // where already applied), plus every other repo; end result matches production. + await setupAllDatabases(); + + const repo = new PersonObservationRepo(); + const row = await repo.upsertByNaturalKey({ + accession_number: "0001-25-000042", + extractor_id: "S-1", + extractor_version: "1.0.0", + observation_index: 0, + last_name: "Smith", + normalized_last: "smith", + titles: ["CEO"], + created_at: "2026-05-22T00:00:00.000Z", + }); + expect(row.titles).toEqual(["CEO"]); + + const readBack = await repo.getById(row.observation_id); + expect(readBack?.titles).toEqual(["CEO"]); + }); +}); diff --git a/src/storage/observation/PersonObservationTitlesMigration.ts b/src/storage/observation/PersonObservationTitlesMigration.ts new file mode 100644 index 00000000..9c786306 --- /dev/null +++ b/src/storage/observation/PersonObservationTitlesMigration.ts @@ -0,0 +1,113 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { globalServiceRegistry } from "workglow"; +import { SEC_DB_FOLDER, SEC_DB_NAME, SEC_DB_TYPE } from "../../config/tokens"; +import { getDb } from "../../util/db"; +import { getPgPool } from "../../util/pg"; + +/** + * Migrates the legacy `person_observations.title TEXT` column to the current + * `titles` shape (SQLite TEXT / Postgres JSONB carrying a JSON string array). + * `SqliteTabularStorage.setupDatabase()` uses `CREATE TABLE IF NOT EXISTS`, so + * an existing DB never gains the renamed column on its own — the first insert + * fails with `table has no column named titles`. + * + * Idempotent: probes the table's column list before adding; a fresh DB (no + * table) is a silent no-op. Existing non-null / non-empty `title` values are + * backfilled as a single-element JSON array; null / empty stay null. The + * legacy `title` column is left in place — dropping it is unnecessary for the + * new writer and keeps rollback trivial. + */ +export async function migratePersonObservationTitles(): Promise { + const dbType = globalServiceRegistry.has(SEC_DB_TYPE) + ? globalServiceRegistry.get(SEC_DB_TYPE) + : null; + + if ( + dbType === "sqlite" && + globalServiceRegistry.has(SEC_DB_FOLDER) && + globalServiceRegistry.has(SEC_DB_NAME) + ) { + return migrateSqlite(); + } + if (dbType === "postgres") { + return migratePostgres(); + } + // In-memory backend: nothing to migrate (tests start with a clean store). +} + +function migrateSqlite(): void { + const db = getDb(); + const tableExistsRow = db + .prepare<[], { name: string }>( + `SELECT name FROM sqlite_master WHERE type='table' AND name='person_observations'` + ) + .get(); + if (!tableExistsRow) return; + const columns = db + .prepare<[], { name: string }>(`PRAGMA table_info(person_observations)`) + .all(); + const hasTitles = columns.some((c) => c.name === "titles"); + if (hasTitles) return; + const hasLegacyTitle = columns.some((c) => c.name === "title"); + db.exec("BEGIN"); + try { + db.exec("ALTER TABLE person_observations ADD COLUMN titles TEXT NULL"); + if (hasLegacyTitle) { + db.exec( + `UPDATE person_observations + SET titles = CASE + WHEN title IS NULL OR title = '' THEN NULL + ELSE json_array(title) + END + WHERE titles IS NULL` + ); + } + db.exec("COMMIT"); + } catch (err) { + db.exec("ROLLBACK"); + throw err; + } +} + +async function migratePostgres(): Promise { + const pool = getPgPool(); + const client = await pool.connect(); + try { + const exists = await client.query( + `SELECT 1 FROM information_schema.tables WHERE table_name = 'person_observations'` + ); + if (exists.rowCount === 0) return; + const cols = await client.query( + `SELECT column_name FROM information_schema.columns WHERE table_name = 'person_observations'` + ); + const names = cols.rows.map((r: { column_name: string }) => r.column_name); + const hasTitles = names.includes("titles"); + if (hasTitles) return; + const hasLegacyTitle = names.includes("title"); + await client.query("BEGIN"); + try { + await client.query(`ALTER TABLE "person_observations" ADD COLUMN "titles" JSONB NULL`); + if (hasLegacyTitle) { + await client.query( + `UPDATE "person_observations" + SET "titles" = CASE + WHEN "title" IS NULL OR "title" = '' THEN NULL + ELSE jsonb_build_array("title") + END + WHERE "titles" IS NULL` + ); + } + await client.query("COMMIT"); + } catch (err) { + await client.query("ROLLBACK"); + throw err; + } + } finally { + client.release(); + } +} diff --git a/src/storage/person/PersonNormalization.test.ts b/src/storage/person/PersonNormalization.test.ts index 91378277..c5711d6c 100644 --- a/src/storage/person/PersonNormalization.test.ts +++ b/src/storage/person/PersonNormalization.test.ts @@ -204,5 +204,43 @@ describe("PersonNormalization", () => { expect(result!.cik).toBe(null); expect(result!.crd).toBe(null); }); + + // Golden pin: the normalizer's output tuple is the resolver's lookup key. + // Any change here silently splits existing canonical rows, so the person + // resolver must go through a major bump ceremony (2.0.0) whenever this + // table changes. + it.each([ + { + input: "Richard J. Boyle, Jr.", + first: "Richard", + middle: "J", + last: "Boyle", + suffix: "Jr", + }, + { + input: "Frank D’Angelo", + first: "Frank", + middle: null as string | null, + last: "D'Angelo", + suffix: null as string | null, + }, + { + input: "Michel del Buono", + first: "Michel", + middle: null as string | null, + last: "del Buono", + suffix: null as string | null, + }, + ])( + "pins the 2.0.0 normalized key tuple for %o", + ({ input, first, middle, last, suffix }) => { + const result = normalizePerson({ name: input }); + expect(result).toBeDefined(); + expect(result!.first).toBe(first); + expect(result!.middle).toBe(middle); + expect(result!.last).toBe(last); + expect(result!.suffix).toBe(suffix); + } + ); }); }); diff --git a/src/storage/spac/SpacLoiColumnsMigration.test.ts b/src/storage/spac/SpacLoiColumnsMigration.test.ts new file mode 100644 index 00000000..b3d7c23b --- /dev/null +++ b/src/storage/spac/SpacLoiColumnsMigration.test.ts @@ -0,0 +1,178 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { mkdtempSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { globalServiceRegistry, Sqlite } from "workglow"; +import { resetDependencyInjectionsForTesting } from "../../config/TestingDI"; +import { setupAllDatabases } from "../../config/setupAllDatabases"; +import { SEC_DB_FOLDER, SEC_DB_NAME, SEC_DB_TYPE } from "../../config/tokens"; +import { closeDb, getDb } from "../../util/db"; +import { migrateSpacLoiColumns } from "./SpacLoiColumnsMigration"; +import { SpacRepo } from "./SpacRepo"; +import type { Spac } from "./SpacSchema"; + +const TEST_DB_NAME = "spac_loi_migration_test"; + +const SPAC_LOI_TABLES = ["spac", "spac_deal", "spac_history"] as const; + +function createLegacySpacTables(db: Sqlite.Database): void { + db.exec( + `CREATE TABLE spac ( + cik INTEGER PRIMARY KEY, + status TEXT NOT NULL, + updated_at TEXT NOT NULL + )` + ); + db.exec( + `CREATE TABLE spac_deal ( + cik INTEGER NOT NULL, + deal_index INTEGER NOT NULL, + outcome TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (cik, deal_index) + )` + ); + db.exec( + `CREATE TABLE spac_history ( + cik INTEGER NOT NULL, + valid_from TEXT NOT NULL, + change_source TEXT NOT NULL, + change_date TEXT NOT NULL, + PRIMARY KEY (cik, valid_from) + )` + ); +} + +describe("migrateSpacLoiColumns (sqlite)", () => { + let tmpDir: string; + + beforeEach(async () => { + resetDependencyInjectionsForTesting(); + closeDb(); + if (typeof Sqlite.init === "function") { + await Sqlite.init(); + } + tmpDir = mkdtempSync(join(tmpdir(), "sec-spac-loi-migration-")); + globalServiceRegistry.registerInstance(SEC_DB_TYPE, "sqlite"); + globalServiceRegistry.registerInstance(SEC_DB_FOLDER, tmpDir); + globalServiceRegistry.registerInstance(SEC_DB_NAME, TEST_DB_NAME); + }); + + afterEach(() => { + closeDb(); + rmSync(tmpDir, { recursive: true, force: true }); + // ServiceRegistry has no unregister API; pin to a non-dispatchable sentinel + // so any subsequent test in the same Bun process falls back to in-memory. + globalServiceRegistry.registerInstance( + SEC_DB_TYPE, + "memory" as unknown as "sqlite" | "postgres" + ); + resetDependencyInjectionsForTesting(); + }); + + it("adds loi_date column to spac, spac_deal, and spac_history", async () => { + const db = getDb(); + createLegacySpacTables(db); + + await migrateSpacLoiColumns(); + + for (const table of SPAC_LOI_TABLES) { + const columns = db + .prepare<[], { name: string }>(`PRAGMA table_info(\`${table}\`)`) + .all(); + expect( + columns.some((c) => c.name === "loi_date"), + `${table} missing loi_date after migration` + ).toBe(true); + } + }); + + it("is idempotent: running twice does not double-add", async () => { + const db = getDb(); + createLegacySpacTables(db); + + await migrateSpacLoiColumns(); + await migrateSpacLoiColumns(); + + for (const table of SPAC_LOI_TABLES) { + const columns = db + .prepare<[], { name: string }>(`PRAGMA table_info(\`${table}\`)`) + .all(); + const loiCols = columns.filter((c) => c.name === "loi_date"); + expect(loiCols).toHaveLength(1); + } + }); + + it("is a no-op when the tables do not exist yet", async () => { + await migrateSpacLoiColumns(); + const db = getDb(); + for (const table of SPAC_LOI_TABLES) { + const row = db + .prepare< + [], + { name: string } + >(`SELECT name FROM sqlite_master WHERE type='table' AND name=?`) + .get(table); + expect(row).toBeUndefined(); + } + }); + + it("SpacRepo.saveSpac succeeds with status='loi' + loi_date after migration", async () => { + const db = getDb(); + createLegacySpacTables(db); + + await setupAllDatabases(); + + const repo = new SpacRepo(); + const row: Spac = { + cik: 9999999, + current_cik: null, + status: "loi", + spac_name: null, + target_name: null, + surviving_name: null, + current_name: null, + spac_sic: null, + post_merger_sic: null, + current_sic: null, + spac_tickers: null, + post_merger_tickers: null, + current_tickers: null, + ipo_proceeds: null, + trust_amount: null, + pipe_amount: null, + total_redemption_amount: null, + focus: null, + focus_location: null, + description: null, + target_description: null, + team: null, + details: null, + url_spac: null, + url_sponsor: null, + investorpres_url: null, + investorpres_date: null, + registration_date: null, + ipo_date: null, + unit_split_date: null, + loi_date: "2026-01-15", + definitive_agreement_date: null, + proxy_date: null, + vote_date: null, + completed_date: null, + failed_date: null, + as_of: null, + updated_at: "2026-05-22T00:00:00.000Z", + }; + await repo.saveSpac(row); + const readBack = await repo.getSpac(9999999); + expect(readBack?.status).toBe("loi"); + expect(readBack?.loi_date).toBe("2026-01-15"); + }); +}); diff --git a/src/storage/spac/SpacLoiColumnsMigration.ts b/src/storage/spac/SpacLoiColumnsMigration.ts new file mode 100644 index 00000000..b0bbac8b --- /dev/null +++ b/src/storage/spac/SpacLoiColumnsMigration.ts @@ -0,0 +1,101 @@ +/** + * @license + * Copyright 2026 Steven Roussey + * SPDX-License-Identifier: Apache-2.0 + */ + +import { globalServiceRegistry } from "workglow"; +import { SEC_DB_FOLDER, SEC_DB_NAME, SEC_DB_TYPE } from "../../config/tokens"; +import { getDb } from "../../util/db"; +import { getPgPool } from "../../util/pg"; + +/** + * Adds the `loi_date` column to `spac`, `spac_deal`, and `spac_history`. The + * LOI lifecycle stage landed as a schema addition, but + * `SqliteTabularStorage.setupDatabase()` uses `CREATE TABLE IF NOT EXISTS`, so + * an existing DB never gains the new column on its own — the first + * `SpacReportWriter.rebuild` after upgrade fails on the `loi_date` column. + * + * The `spac.status` enum also gained the value `"loi"`, but `TypeStringEnum` + * generates plain TEXT with **no CHECK constraint** (the enum is validated at + * the schema layer, not the database), so no DDL change is required for + * `status`. This migration is limited to the three `loi_date` columns. + * + * Idempotent: probes each table's column list before adding; a fresh DB (no + * tables) is a silent no-op. + */ +export async function migrateSpacLoiColumns(): Promise { + const dbType = globalServiceRegistry.has(SEC_DB_TYPE) + ? globalServiceRegistry.get(SEC_DB_TYPE) + : null; + + if ( + dbType === "sqlite" && + globalServiceRegistry.has(SEC_DB_FOLDER) && + globalServiceRegistry.has(SEC_DB_NAME) + ) { + return migrateSqlite(); + } + if (dbType === "postgres") { + return migratePostgres(); + } + // In-memory backend: nothing to migrate (tests start with a clean store). +} + +const SPAC_LOI_TABLES = ["spac", "spac_deal", "spac_history"] as const; + +function migrateSqlite(): void { + const db = getDb(); + db.exec("BEGIN"); + try { + for (const table of SPAC_LOI_TABLES) { + const tableExistsRow = db + .prepare<[], { name: string }>( + `SELECT name FROM sqlite_master WHERE type='table' AND name=?` + ) + .get(table); + if (!tableExistsRow) continue; + const columns = db + .prepare<[], { name: string }>(`PRAGMA table_info(\`${table}\`)`) + .all(); + if (columns.some((c) => c.name === "loi_date")) continue; + db.exec(`ALTER TABLE \`${table}\` ADD COLUMN loi_date TEXT NULL`); + } + db.exec("COMMIT"); + } catch (err) { + db.exec("ROLLBACK"); + throw err; + } +} + +async function migratePostgres(): Promise { + const pool = getPgPool(); + const client = await pool.connect(); + try { + await client.query("BEGIN"); + try { + for (const table of SPAC_LOI_TABLES) { + const exists = await client.query( + `SELECT 1 FROM information_schema.tables WHERE table_name = $1`, + [table] + ); + if (exists.rowCount === 0) continue; + const cols = await client.query( + `SELECT column_name FROM information_schema.columns WHERE table_name = $1`, + [table] + ); + const hasLoiDate = cols.rows.some( + (r: { column_name: string }) => r.column_name === "loi_date" + ); + if (hasLoiDate) continue; + await client.query(`ALTER TABLE "${table}" ADD COLUMN "loi_date" DATE NULL`); + } + await client.query("COMMIT"); + } catch (err) { + await client.query("ROLLBACK"); + throw err; + } + } finally { + client.release(); + } +} diff --git a/src/storage/versioning/bootstrapComponentVersions.test.ts b/src/storage/versioning/bootstrapComponentVersions.test.ts index 387c3d69..7bf205d3 100644 --- a/src/storage/versioning/bootstrapComponentVersions.test.ts +++ b/src/storage/versioning/bootstrapComponentVersions.test.ts @@ -9,9 +9,14 @@ import { globalServiceRegistry } from "workglow"; import { resetDependencyInjectionsForTesting } from "../../config/TestingDI"; import { setupAllDatabases } from "../../config/setupAllDatabases"; import { bootstrapComponentVersions } from "./bootstrapComponentVersions"; +import { promote, startDev } from "./ceremonies"; import { COMPONENT_VERSION_REPOSITORY_TOKEN } from "./ComponentVersionSchema"; import { EXTRACTOR_IDS } from "./extractorIds"; +import { ExtractorRunRepo } from "./ExtractorRunRepo"; +import { EXTRACTOR_RUN_REPOSITORY_TOKEN } from "./ExtractorRunSchema"; import { RESOLVER_IDS } from "../../resolver/resolverIds"; +import { VersionEventRepo } from "./VersionEventRepo"; +import { VERSION_EVENT_REPOSITORY_TOKEN } from "./VersionEventSchema"; import { VersionRegistry } from "./VersionRegistry"; describe("bootstrapComponentVersions", () => { @@ -106,4 +111,48 @@ describe("bootstrapComponentVersions", () => { expect(await reg.getPrevious("extractor", "D")).toBeUndefined(); expect(await reg.getNext("extractor", "D")).toBeUndefined(); }); + + it("startDev+promote for resolver:person rotates 1.0.0 → previous, 2.0.0 → current", async () => { + const reg = new VersionRegistry( + globalServiceRegistry.get(COMPONENT_VERSION_REPOSITORY_TOKEN) + ); + const events = new VersionEventRepo( + globalServiceRegistry.get(VERSION_EVENT_REPOSITORY_TOKEN) + ); + const runs = new ExtractorRunRepo( + globalServiceRegistry.get(EXTRACTOR_RUN_REPOSITORY_TOKEN) + ); + + // bootstrapComponentVersions seeded resolver:person@1.0.0 already. + expect((await reg.getCurrent("resolver", "person"))?.semver).toBe("1.0.0"); + + await startDev({ + reg, + events, + kind: "resolver", + id: "person", + semver: "2.0.0", + bump: "major", + targetCount: 0, // no observations seeded → coverage is 0/0 → fraction 0 (not 1) + notes: "PersonNormalization fold changes key tuple", + }); + // Both slots exist. + expect((await reg.getCurrent("resolver", "person"))?.semver).toBe("1.0.0"); + expect((await reg.getNext("resolver", "person"))?.semver).toBe("2.0.0"); + + // 0/0 coverage → --force required to promote. + await promote({ + reg, + events, + runs, + kind: "resolver", + id: "person", + force: true, + notes: null, + }); + + expect((await reg.getCurrent("resolver", "person"))?.semver).toBe("2.0.0"); + expect((await reg.getPrevious("resolver", "person"))?.semver).toBe("1.0.0"); + expect(await reg.getNext("resolver", "person")).toBeUndefined(); + }); }); diff --git a/src/storage/versioning/ceremonies.test.ts b/src/storage/versioning/ceremonies.test.ts index f17deec5..090266e1 100644 --- a/src/storage/versioning/ceremonies.test.ts +++ b/src/storage/versioning/ceremonies.test.ts @@ -17,6 +17,7 @@ import { VERSION_EVENT_REPOSITORY_TOKEN } from "./VersionEventSchema"; import { VersionRegistry } from "./VersionRegistry"; import { PersonIdentityLinkRepo } from "../canonical/PersonIdentityLinkRepo"; import { CanonicalPersonRepo } from "../canonical/CanonicalPersonRepo"; +import { PersonObservationRepo } from "../observation/PersonObservationRepo"; function buildDeps() { const reg = new VersionRegistry( @@ -730,6 +731,136 @@ describe("ceremonies.dropPrevious", () => { expect(dropEvt?.notes).toBe("cleaning up 1.0.0"); }); + it("resolver major promote fails when coverage is below 100% without --force", async () => { + const { reg, events, runs } = buildDeps(); + // Seed an observation so the denominator is > 0 and no identity-link at + // 2.0.0 has been written yet — coverage = 0/1. + const obsRepo = new PersonObservationRepo(); + await obsRepo.upsertByNaturalKey({ + accession_number: "0001-25-000001", + extractor_id: "D", + extractor_version: "1.0.0", + observation_index: 0, + last_name: "Smith", + normalized_last: "smith", + created_at: "2026-05-22T00:00:00.000Z", + }); + + await startDev({ + reg, + events, + kind: "resolver", + id: "person", + semver: "2.0.0", + bump: "major", + targetCount: 1, + notes: null, + }); + await expect( + promote({ + reg, + events, + runs, + kind: "resolver", + id: "person", + force: false, + notes: null, + }) + ).rejects.toThrow(/coverage.*0\/1/i); + }); + + it("resolver major promote succeeds at 100% coverage", async () => { + const { reg, events, runs } = buildDeps(); + const obsRepo = new PersonObservationRepo(); + const linkRepo = new PersonIdentityLinkRepo(); + const canonRepo = new CanonicalPersonRepo(); + + const obs = await obsRepo.upsertByNaturalKey({ + accession_number: "0001-25-000001", + extractor_id: "D", + extractor_version: "1.0.0", + observation_index: 0, + last_name: "Smith", + normalized_last: "smith", + created_at: "2026-05-22T00:00:00.000Z", + }); + // Seed a canonical + identity link at 2.0.0 so coverage = 1/1. + const canonId = "00000000-0000-0000-0000-00000000cafe"; + await canonRepo.create({ + canonical_person_id: canonId, + resolver_version: "2.0.0", + display_first: null, + display_middle: null, + display_last: "Smith", + display_suffix: null, + cik: null, + normalized_first: null, + normalized_middle: null, + normalized_last: "smith", + normalized_suffix: null, + source_filing_issuer_cik: null, + created_at: new Date().toISOString(), + }); + await linkRepo.upsert(obs.observation_id, "2.0.0", canonId); + + await startDev({ + reg, + events, + kind: "resolver", + id: "person", + semver: "2.0.0", + bump: "major", + targetCount: 1, + notes: null, + }); + await promote({ + reg, + events, + runs, + kind: "resolver", + id: "person", + force: false, + notes: null, + }); + expect((await reg.getCurrent("resolver", "person"))?.semver).toBe("2.0.0"); + expect((await reg.getPrevious("resolver", "person"))?.semver).toBe("1.0.0"); + }); + + it("resolver major promote --force bypasses the coverage gate", async () => { + const { reg, events, runs } = buildDeps(); + const obsRepo = new PersonObservationRepo(); + await obsRepo.upsertByNaturalKey({ + accession_number: "0001-25-000001", + extractor_id: "D", + extractor_version: "1.0.0", + observation_index: 0, + last_name: "Smith", + normalized_last: "smith", + created_at: "2026-05-22T00:00:00.000Z", + }); + + await startDev({ + reg, + events, + kind: "resolver", + id: "person", + semver: "2.0.0", + bump: "major", + targetCount: 1, + notes: null, + }); + await promote({ + reg, + events, + runs, + kind: "resolver", + id: "person", + force: true, + notes: "force-promoting without coverage", + }); + expect((await reg.getCurrent("resolver", "person"))?.semver).toBe("2.0.0"); + }); + it("dropPrevious(extractor) deletes run rows at previous semver", async () => { const { reg, events, runs } = buildDeps(); diff --git a/src/storage/versioning/ceremonies.ts b/src/storage/versioning/ceremonies.ts index 025ba9d0..ffe27fe2 100644 --- a/src/storage/versioning/ceremonies.ts +++ b/src/storage/versioning/ceremonies.ts @@ -19,6 +19,7 @@ import { CanonicalPersonPhoneRepo } from "../canonical/CanonicalPersonPhoneRepo" import { CanonicalCompanyAddressRepo } from "../canonical/CanonicalCompanyAddressRepo"; import { CanonicalCompanyPhoneRepo } from "../canonical/CanonicalCompanyPhoneRepo"; import type { ResolverId } from "../../resolver/resolverIds"; +import { computeResolverCoverage } from "../../cli/queries/ResolverCoverage"; interface BaseArgs { readonly reg: VersionRegistry; @@ -37,6 +38,7 @@ export interface StartDevArgs extends BaseArgs { } export interface PromoteArgs extends BaseArgs { + /** Required when `kind === "extractor"`. Unused for resolvers. */ readonly runs: ExtractorRunRepo; readonly force: boolean; readonly notes: string | null; @@ -171,19 +173,31 @@ export async function promote(args: PromoteArgs): Promise { const current = await reg.getCurrent(kind, id); if (!current) throw new Error(`No current slot for ${kind} '${id}'.`); - // Major coverage gate. + // Major coverage gate. Dispatch by kind: extractors use extractor_runs + // count; resolvers use identity-link-over-observation coverage. if (next.bump_type === "major" && !force) { if (next.target_count === null) { throw new Error( `${kind} '${id}' next slot has bump_type='major' but target_count is null — invalid state. Drop-next and re-run start-dev.` ); } - const target = next.target_count; - const successful = await runs.countSuccessfulAtVersion(id, next.semver); - if (successful < target) { - throw new Error( - `coverage ${successful}/${target} below 100% — use --force to promote anyway` - ); + if (kind === "extractor") { + const target = next.target_count; + const successful = await runs.countSuccessfulAtVersion(id, next.semver); + if (successful < target) { + throw new Error( + `coverage ${successful}/${target} below 100% — use --force to promote anyway` + ); + } + } else { + // resolver — computeResolverCoverage throws for family-tier resolvers, + // so promote is refused for them until family coverage lands. + const coverage = await computeResolverCoverage(id as ResolverId, next.semver); + if (coverage.fraction < 1.0) { + throw new Error( + `coverage ${coverage.numerator}/${coverage.denominator} below 100% — use --force to promote anyway` + ); + } } }