From 4f2542dcc11cde47494b7fb77d277b64a10fb9ba Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Tue, 14 Jul 2026 16:12:28 +1000 Subject: [PATCH 1/6] feat(migrate): detect a column's EQL version (v2 vs v3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of EQL v3 support (#648): detectColumnEqlVersion() inspects an encrypted column's Postgres domain type — public.eql_v2_encrypted -> v2, a concrete eql_v3_* domain -> v3, anything else (plaintext / not found) -> null. This is the keystone the version-aware lifecycle branches on. Unit-tested with a mocked pg client; no behaviour change to existing v2 paths. Refs #648 Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .../migrate/src/__tests__/version.test.ts | 72 +++++++++++++++++++ packages/migrate/src/index.ts | 1 + packages/migrate/src/version.ts | 48 +++++++++++++ 3 files changed, 121 insertions(+) create mode 100644 packages/migrate/src/__tests__/version.test.ts create mode 100644 packages/migrate/src/version.ts diff --git a/packages/migrate/src/__tests__/version.test.ts b/packages/migrate/src/__tests__/version.test.ts new file mode 100644 index 000000000..e95036cec --- /dev/null +++ b/packages/migrate/src/__tests__/version.test.ts @@ -0,0 +1,72 @@ +import type { ClientBase, QueryConfig, QueryResult, QueryResultRow } from 'pg' +import { describe, expect, it } from 'vitest' +import { detectColumnEqlVersion } from '../version.js' + +interface RecordedQuery { + text: string + values: unknown[] +} + +function createMockClient(rows: Array>): { + client: ClientBase + queries: RecordedQuery[] +} { + const queries: RecordedQuery[] = [] + const client = { + query(config: string | QueryConfig, values?: unknown[]) { + const text = typeof config === 'string' ? config : config.text + queries.push({ text, values: values ?? [] }) + return Promise.resolve({ + rows, + rowCount: rows.length, + command: '', + oid: 0, + fields: [], + } as unknown as QueryResult) + }, + } as unknown as ClientBase + return { client, queries } +} + +describe('detectColumnEqlVersion', () => { + it('maps the eql_v2_encrypted domain to v2', async () => { + const { client } = createMockClient([{ domain_name: 'eql_v2_encrypted' }]) + expect( + await detectColumnEqlVersion(client, 'users', 'email_encrypted'), + ).toBe('v2') + }) + + it('maps any concrete eql_v3_* domain to v3', async () => { + for (const domain of [ + 'eql_v3_text_search', + 'eql_v3_text_match', + 'eql_v3_int8_ord', + 'eql_v3_encrypted', + ]) { + const { client } = createMockClient([{ domain_name: domain }]) + expect( + await detectColumnEqlVersion(client, 'users', 'email_encrypted'), + ).toBe('v3') + } + }) + + it('returns null for a plaintext column (base type, not a domain)', async () => { + const { client } = createMockClient([{ domain_name: 'text' }]) + expect(await detectColumnEqlVersion(client, 'users', 'email')).toBeNull() + }) + + it('returns null when the column/table is not found', async () => { + const { client } = createMockClient([]) + expect(await detectColumnEqlVersion(client, 'nope', 'missing')).toBeNull() + }) + + it('passes the qualified table name and column as bind params (to_regclass)', async () => { + const { client, queries } = createMockClient([ + { domain_name: 'eql_v3_text_search' }, + ]) + await detectColumnEqlVersion(client, 'app.users', 'email_encrypted') + expect(queries).toHaveLength(1) + expect(queries[0].text).toContain('to_regclass($1)') + expect(queries[0].values).toEqual(['app.users', 'email_encrypted']) + }) +}) diff --git a/packages/migrate/src/index.ts b/packages/migrate/src/index.ts index 5c3620d8f..dfaec9e65 100644 --- a/packages/migrate/src/index.ts +++ b/packages/migrate/src/index.ts @@ -67,3 +67,4 @@ export { type MigrationStateRow, progress, } from './state.js' +export { detectColumnEqlVersion, type EqlVersion } from './version.js' diff --git a/packages/migrate/src/version.ts b/packages/migrate/src/version.ts new file mode 100644 index 000000000..16ad7fd58 --- /dev/null +++ b/packages/migrate/src/version.ts @@ -0,0 +1,48 @@ +import type { ClientBase } from 'pg' + +/** + * Which EQL generation an encrypted column belongs to. The migration lifecycle + * differs between them: v2 is driven by the `eql_v2_configuration` state machine + * (see {@link import('./eql.js')}), while v3 is domain-native — configuration + * lives in the column's own type and there is no configuration table, so its + * lifecycle is backfill-then-drop with no cut-over rename. + */ +export type EqlVersion = 'v2' | 'v3' + +/** + * Detect the EQL version of a column by inspecting its Postgres type. + * + * - v2 encrypted columns are the `public.eql_v2_encrypted` domain. + * - v3 encrypted columns are a concrete `eql_v3_*` domain (e.g. + * `eql_v3_text_search`, `eql_v3_int8_ord`). + * - Anything else — a plaintext column, or a column/table that doesn't exist — + * returns `null`. + * + * Pass the **encrypted target** column (e.g. `email_encrypted`), not the + * plaintext source: it's the encrypted column whose domain type carries the EQL + * generation. `tableName` may be schema-qualified (`"schema.table"`); + * resolution honours the connection's `search_path` via `to_regclass`. + */ +export async function detectColumnEqlVersion( + client: ClientBase, + tableName: string, + columnName: string, +): Promise { + // `a.atttypid` on a domain-typed column is the DOMAIN's oid, so `t.typname` + // is the domain name (e.g. `eql_v2_encrypted`), not the underlying `jsonb`. + // `to_regclass` returns NULL for an unknown table → no rows → null. + const result = await client.query<{ domain_name: string }>( + `SELECT t.typname AS domain_name + FROM pg_attribute a + JOIN pg_type t ON t.oid = a.atttypid + WHERE a.attrelid = to_regclass($1) + AND a.attname = $2 + AND NOT a.attisdropped`, + [tableName, columnName], + ) + const domain = result.rows[0]?.domain_name + if (domain === undefined) return null + if (domain === 'eql_v2_encrypted') return 'v2' + if (domain.startsWith('eql_v3')) return 'v3' + return null +} From 17b46e7552f7409c72f495f0425a671f1defd391 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Thu, 16 Jul 2026 18:23:42 +1000 Subject: [PATCH 2/6] =?UTF-8?q?feat(migrate,cli):=20EQL=20v3=20rollout=20l?= =?UTF-8?q?ifecycle=20=E2=80=94=20phases=202-5=20(#648)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes #649. The backfill engine was already version-agnostic (it never touches the v2 config machine in eql.ts — that's CLI-called only), and isEncryptedPayload already accepts both v3 wire shapes, so the work lands where the v2 coupling actually lived: migrate: - countEncrypted(client, table, column) — the v3 verification primitive (v3 has no eql_v2.count_encrypted_with_active_config; a plain count of the populated domain column is the equivalent, the domain CHECK already guarantees validity). - Manifest gains an optional eqlVersion (2|3) field, recorded at backfill time, so status/plan branch without a DB round-trip. - backfill-v3.integration.test.ts pins the v3 contract against a real Postgres: flat-scalar and SteVec payloads through the leak guard, the $N::jsonb write, countEncrypted, idempotency. vitest fileParallelism disabled for the package — the two integration suites share the singleton cipherstash.cs_migrations schema and raced when parallel. cli (all auto-detected via detectColumnEqlVersion, no new flags): - encrypt backfill: detects the version, logs the lifecycle, records eqlVersion + the v3 target phase ('dropped' — no cut-over) in the manifest, and prints v3 next steps (switch-by-name, then drop). - encrypt cutover: v3 short-circuits with "not applicable" guidance (exit 0) before any v2 config-machine call. - encrypt drop: v3 runs from 'backfilled' and drops the ORIGINAL column (no _plaintext exists under v3); v2 unchanged, both pinned by tests. - encrypt status: v2-only drift flags (not-registered, plaintext-col-missing) suppressed for v3 columns; EQL column shows 'v3'. - Fixed a pre-existing exit-code bug the new tests exposed: cutover/drop precondition guards `return` from inside `try`, so the trailing `if (exitCode) process.exit(...)` was unreachable — precondition failures exited 0. The exit now lives in `finally`. Verified live against postgres-eql + EQL v3 installed by this branch's CLI: real EncryptionV3 ciphertext backfilled into a concrete eql_v3_text domain column (CHECK satisfied), detectColumnEqlVersion → v3, countEncrypted exact, decrypt round-trip exact. Docs: migrate README rewritten around the two lifecycles; stash-cli and stash-encryption skills updated (the #648 v2-only notes are gone). Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .changeset/migrate-eql-v3.md | 34 ++++ packages/cli/src/bin/main.ts | 2 +- packages/cli/src/cli/registry.ts | 2 +- .../encrypt/__tests__/encrypt-v3.test.ts | 162 +++++++++++++++ packages/cli/src/commands/encrypt/backfill.ts | 32 ++- packages/cli/src/commands/encrypt/cutover.ts | 28 ++- packages/cli/src/commands/encrypt/drop.ts | 40 +++- packages/cli/src/commands/encrypt/status.ts | 21 +- packages/migrate/README.md | 20 +- .../__tests__/backfill-v3.integration.test.ts | 192 ++++++++++++++++++ packages/migrate/src/cursor.ts | 23 +++ packages/migrate/src/index.ts | 1 + packages/migrate/src/manifest.ts | 9 + packages/migrate/vitest.config.ts | 12 ++ skills/stash-cli/SKILL.md | 8 +- skills/stash-encryption/SKILL.md | 4 +- 16 files changed, 564 insertions(+), 26 deletions(-) create mode 100644 .changeset/migrate-eql-v3.md create mode 100644 packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts create mode 100644 packages/migrate/src/__tests__/backfill-v3.integration.test.ts create mode 100644 packages/migrate/vitest.config.ts diff --git a/.changeset/migrate-eql-v3.md b/.changeset/migrate-eql-v3.md new file mode 100644 index 000000000..08ef82026 --- /dev/null +++ b/.changeset/migrate-eql-v3.md @@ -0,0 +1,34 @@ +--- +'@cipherstash/migrate': minor +'stash': minor +--- + +EQL v3 support for the encryption rollout lifecycle (#648). The `stash +encrypt *` commands (and `@cipherstash/migrate` underneath) now auto-detect a +column's EQL version from its Postgres domain type and follow the right +lifecycle — no new flags: + +- **`encrypt backfill`** works on v3 columns unchanged (the engine was always + version-agnostic; pass an `EncryptionV3` client and real v3 envelopes land + in the concrete `eql_v3_*` domain column — verified live against a real + database, including the domain CHECK and a decrypt round-trip). The + manifest records the detected version and the v3 target phase, and the + command prints v3-appropriate next steps. +- **`encrypt cutover`** on a v3 column reports "not applicable" (exit 0) with + guidance: v3 has no rename cut-over — the application switches to + `_encrypted` by name. +- **`encrypt drop`** is version-aware: v3 runs from the `backfilled` phase + and drops the ORIGINAL plaintext column (there is no `_plaintext` + under v3); v2 behaviour is unchanged. +- **`encrypt status`** no longer raises the v2-only `not-registered` / + `plaintext-col-missing` drift flags for v3 columns (v3 has no + `eql_v2_configuration` and no rename) and shows `v3` in the EQL column. +- New `@cipherstash/migrate` exports: `countEncrypted` (the v3 + backfill-verification primitive) and a manifest `eqlVersion` field. +- Fixed: `encrypt cutover`/`encrypt drop` precondition failures now actually + exit 1 — the early-return guards previously skipped the exit-code path + entirely, so failed preconditions exited 0. + +The `stash-cli` and `stash-encryption` skills and the `@cipherstash/migrate` +README document the two lifecycles (v2: backfill → cutover → drop; +v3: backfill → switch-by-name → drop). diff --git a/packages/cli/src/bin/main.ts b/packages/cli/src/bin/main.ts index e4f6306b8..515423778 100644 --- a/packages/cli/src/bin/main.ts +++ b/packages/cli/src/bin/main.ts @@ -122,7 +122,7 @@ Commands: encrypt status Show per-column migration status (phase, progress, drift) encrypt plan Diff intent (.cipherstash/migrations.json) vs observed state encrypt backfill Resumably encrypt plaintext into the encrypted column - encrypt cutover Rename swap encrypted → primary column + encrypt cutover Rename swap encrypted → primary column (EQL v2 only) encrypt drop Generate a migration to drop the plaintext column env (experimental) Print production env vars for deployment diff --git a/packages/cli/src/cli/registry.ts b/packages/cli/src/cli/registry.ts index 66f527f5a..3c1fb8122 100644 --- a/packages/cli/src/cli/registry.ts +++ b/packages/cli/src/cli/registry.ts @@ -498,7 +498,7 @@ export const registry: CommandGroup[] = [ }, { name: 'encrypt cutover', - summary: 'Rename swap encrypted → primary column', + summary: 'Rename swap encrypted → primary column (EQL v2 only)', flags: [ TABLE_FLAG, COLUMN_FLAG, diff --git a/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts b/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts new file mode 100644 index 000000000..9e48bfc0a --- /dev/null +++ b/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts @@ -0,0 +1,162 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +// The EQL-v3 lifecycle branches (#648/#649): v3 has no cut-over rename — +// `encrypt cutover` must short-circuit with guidance instead of running the +// v2 config machine, and `encrypt drop` must target the ORIGINAL plaintext +// column (there is no `_plaintext`) gated on `backfilled` rather than +// `cut-over`. The v2 paths are pinned alongside as regression guards. + +const queryMock = vi.hoisted(() => vi.fn(async () => ({ rows: [] }))) +vi.mock('pg', () => ({ + default: { + Client: class { + connect = vi.fn(async () => {}) + end = vi.fn(async () => {}) + query = queryMock + }, + }, +})) + +const migrateMocks = vi.hoisted(() => ({ + detectColumnEqlVersion: vi.fn(async () => 'v3' as 'v2' | 'v3' | null), + progress: vi.fn( + async () => ({ phase: 'backfilled' }) as { phase: string } | null, + ), + appendEvent: vi.fn(async () => {}), + setManifestTargetPhase: vi.fn(async () => {}), + renameEncryptedColumns: vi.fn(async () => {}), + migrateConfig: vi.fn(async () => {}), + activateConfig: vi.fn(async () => {}), + reloadConfig: vi.fn(async () => {}), +})) +vi.mock('@cipherstash/migrate', () => migrateMocks) + +vi.mock('@clack/prompts', () => ({ + intro: vi.fn(), + outro: vi.fn(), + confirm: vi.fn(async () => true), + isCancel: vi.fn(() => false), + log: { + info: vi.fn(), + success: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + step: vi.fn(), + }, + note: vi.fn(), +})) +vi.mock('@/config/index.js', () => ({ + loadStashConfig: vi.fn(async () => ({ databaseUrl: 'postgres://test' })), +})) +vi.mock('@/commands/db/detect.js', () => ({ + detectDrizzle: vi.fn(() => false), +})) +vi.mock('@/commands/init/utils.js', () => ({ + detectPackageManager: vi.fn(() => 'npm'), + runnerCommand: vi.fn((_pm: string, ref: string) => `npx ${ref}`), +})) +const scaffoldMock = vi.hoisted(() => + vi.fn(async ({ name }: { name: string }) => ({ + path: `drizzle/${name}.sql`, + })), +) +vi.mock('../drizzle-helper.js', () => ({ + scaffoldDrizzleMigration: scaffoldMock, +})) +const writeFileMock = vi.hoisted(() => vi.fn()) +vi.mock('node:fs', () => ({ + default: { mkdirSync: vi.fn(), writeFileSync: writeFileMock }, +})) + +import * as p from '@clack/prompts' +import { cutoverCommand } from '../cutover.js' +import { dropCommand } from '../drop.js' + +describe('encrypt cutover — EQL version awareness', () => { + beforeEach(() => { + vi.clearAllMocks() + migrateMocks.progress.mockResolvedValue({ phase: 'backfilled' }) + }) + + it('short-circuits on a v3 column: guidance, no rename, no config machine', async () => { + migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v3') + + await cutoverCommand({ table: 'users', column: 'email' }) + + expect(p.log.info).toHaveBeenCalledWith( + expect.stringContaining('not applicable to EQL v3'), + ) + expect(p.log.info).toHaveBeenCalledWith( + expect.stringContaining('stash encrypt drop'), + ) + expect(migrateMocks.renameEncryptedColumns).not.toHaveBeenCalled() + expect(migrateMocks.migrateConfig).not.toHaveBeenCalled() + expect(migrateMocks.activateConfig).not.toHaveBeenCalled() + expect(migrateMocks.appendEvent).not.toHaveBeenCalled() + }) + + it('still runs the v2 flow for a v2 column (regression pin)', async () => { + migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v2') + // pending-config check returns true + queryMock.mockImplementation(async (sql: string) => + typeof sql === 'string' && sql.includes('eql_v2_configuration') + ? { rows: [{ exists: true }] } + : { rows: [] }, + ) + + await cutoverCommand({ table: 'users', column: 'email' }) + + expect(migrateMocks.renameEncryptedColumns).toHaveBeenCalled() + expect(migrateMocks.migrateConfig).toHaveBeenCalled() + expect(migrateMocks.activateConfig).toHaveBeenCalled() + }) +}) + +describe('encrypt drop — EQL version awareness', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('v3: requires phase backfilled and drops the ORIGINAL column', async () => { + migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v3') + migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilled' }) + + await dropCommand({ table: 'users', column: 'email' }) + + // Non-drizzle fallback writes the SQL file — inspect the generated DDL. + const sql = writeFileMock.mock.calls[0]?.[1] as string + expect(sql).toContain('DROP COLUMN "email"') + expect(sql).not.toContain('email_plaintext') + expect(migrateMocks.appendEvent).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ event: 'dropped', phase: 'dropped' }), + ) + }) + + it('v3: rejects when not yet backfilled', async () => { + migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v3') + migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilling' }) + const exitSpy = vi + .spyOn(process, 'exit') + .mockImplementation((() => undefined) as never) + + await dropCommand({ table: 'users', column: 'email' }) + + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining("Must be 'backfilled'"), + ) + expect(writeFileMock).not.toHaveBeenCalled() + expect(exitSpy).toHaveBeenCalledWith(1) + exitSpy.mockRestore() + }) + + it('v2: unchanged — requires cut-over and drops _plaintext (regression pin)', async () => { + migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v2') + migrateMocks.progress.mockResolvedValueOnce({ phase: 'cut-over' }) + + await dropCommand({ table: 'users', column: 'email' }) + + const sql = writeFileMock.mock.calls[0]?.[1] as string + expect(sql).toContain('DROP COLUMN "email_plaintext"') + }) +}) diff --git a/packages/cli/src/commands/encrypt/backfill.ts b/packages/cli/src/commands/encrypt/backfill.ts index 84b2ceab8..7051ad89e 100644 --- a/packages/cli/src/commands/encrypt/backfill.ts +++ b/packages/cli/src/commands/encrypt/backfill.ts @@ -1,5 +1,6 @@ import { appendEvent, + detectColumnEqlVersion, type ManifestColumn, progress, runBackfill, @@ -136,6 +137,22 @@ export async function backfillCommand(options: BackfillCommandOptions) { const encryptedColumn = options.encryptedColumn ?? `${options.column}_encrypted` + // v2 or v3 changes the rest of the LIFECYCLE (v3 has no cut-over — the + // ladder is backfill → switch-by-name → drop), so detect it up front, + // record it in the manifest, and tell the user which path they're on. + // `null` means the target column doesn't exist or isn't an EQL domain — + // let the existing checks below produce their specific errors. + const eqlVersion = await detectColumnEqlVersion( + db, + options.table, + encryptedColumn, + ) + if (eqlVersion) { + p.log.info( + `${options.table}.${encryptedColumn} is EQL ${eqlVersion}${eqlVersion === 'v3' ? ' — lifecycle is backfill → switch the app to the encrypted column by name → drop (no cut-over rename).' : ''}`, + ) + } + // Phase guard: backfill requires the application to already be writing // to both columns, otherwise rows inserted *during* the backfill land // in plaintext only and create silent migration drift. Records the @@ -186,6 +203,7 @@ export async function backfillCommand(options: BackfillCommandOptions) { schemaColumnKey, plaintextColumn, options.pkColumn, + eqlVersion, ) await upsertManifestColumn(options.table, manifestEntry) p.log.success( @@ -254,6 +272,12 @@ export async function backfillCommand(options: BackfillCommandOptions) { return } + if (eqlVersion === 'v3') { + p.note( + `EQL v3 has no cut-over. Next:\n 1. Point your application at ${encryptedColumn} (schema + queries), deploy, verify reads.\n 2. Generate the plaintext drop: stash encrypt drop --table ${options.table} --column ${plaintextColumn}`, + 'Next steps (EQL v3)', + ) + } p.outro( `Backfill complete. ${result.rowsProcessed.toLocaleString()} rows encrypted.`, ) @@ -499,6 +523,7 @@ function buildManifestEntry( schemaColumnKey: string, plaintextColumn: string, pkColumn: string | undefined, + eqlVersion: 'v2' | 'v3' | null, ): ManifestColumn { // SDK `cast_as` ('string', 'number', …) and EQL `castAs` ('text', // 'double', …) are different vocabularies; translate via the same @@ -520,9 +545,12 @@ function buildManifestEntry( column: plaintextColumn, castAs, indexes, - targetPhase: 'cut-over', + // v2's ladder ends with the rename cut-over; v3 has none — its end + // state is the plaintext column dropped. + targetPhase: eqlVersion === 'v3' ? 'dropped' : 'cut-over', ...(pkColumn ? { pkColumn } : {}), - } + ...(eqlVersion ? { eqlVersion: eqlVersion === 'v3' ? 3 : 2 } : {}), + } as ManifestColumn } // Drop the wrapping default so unknown values fail validation instead of diff --git a/packages/cli/src/commands/encrypt/cutover.ts b/packages/cli/src/commands/encrypt/cutover.ts index 3d8d4c816..733d6bb71 100644 --- a/packages/cli/src/commands/encrypt/cutover.ts +++ b/packages/cli/src/commands/encrypt/cutover.ts @@ -1,6 +1,7 @@ import { activateConfig, appendEvent, + detectColumnEqlVersion, migrateConfig, progress, reloadConfig, @@ -62,6 +63,26 @@ export async function cutoverCommand(options: CutoverCommandOptions) { try { await client.connect() + // Cut-over is an EQL v2 concept: v2 hides the swap behind + // `eql_v2.rename_encrypted_columns()` + a Proxy config promotion. A v3 + // column has neither — the application switches to the encrypted column + // BY NAME, and the plaintext column is dropped later. Detect before any + // phase/config checks so v3 users get the real answer, not a confusing + // precondition error. + const encryptedColumn = `${options.column}_encrypted` + const version = await detectColumnEqlVersion( + client, + options.table, + encryptedColumn, + ) + if (version === 'v3') { + p.log.info( + `Cut-over is not applicable to EQL v3 columns. ${options.table}.${encryptedColumn} is EQL v3: there is no rename step — point your application at ${encryptedColumn} (update your schema/queries), verify reads, then generate the plaintext drop with:\n stash encrypt drop --table ${options.table} --column ${options.column}`, + ) + p.outro('Nothing to do for EQL v3.') + return + } + const state = await progress(client, options.table, options.column) if (state?.phase !== 'backfilled') { p.log.error( @@ -183,8 +204,13 @@ export async function cutoverCommand(options: CutoverCommandOptions) { exitCode = 1 } finally { await client.end() + // In `finally` (not after the try/catch) deliberately: the precondition + // guards above `return` from inside `try`, which skips any code placed + // after the block — so a trailing `if (exitCode) process.exit(...)` + // was unreachable on exactly the failure paths it existed for, and + // guard failures exited 0. + if (exitCode) process.exit(exitCode) } - if (exitCode) process.exit(exitCode) } /** diff --git a/packages/cli/src/commands/encrypt/drop.ts b/packages/cli/src/commands/encrypt/drop.ts index 8f75034e8..b385235d8 100644 --- a/packages/cli/src/commands/encrypt/drop.ts +++ b/packages/cli/src/commands/encrypt/drop.ts @@ -2,6 +2,7 @@ import fs from 'node:fs' import path from 'node:path' import { appendEvent, + detectColumnEqlVersion, progress, setManifestTargetPhase, } from '@cipherstash/migrate' @@ -58,21 +59,43 @@ export async function dropCommand(options: DropCommandOptions) { try { await client.connect() + // The plaintext column's name depends on the EQL version's lifecycle: + // v2's cut-over renames `` → `_plaintext` (so that's what we + // drop, after `cut-over`); v3 has no rename — the app switched to + // `_encrypted` by name, so the original `` IS the plaintext + // column, droppable straight after `backfilled`. + const version = await detectColumnEqlVersion( + client, + options.table, + `${options.column}_encrypted`, + ) + const isV3 = version === 'v3' + const requiredPhase = isV3 ? 'backfilled' : 'cut-over' + const plaintextToDrop = isV3 + ? options.column + : `${options.column}_plaintext` + const state = await progress(client, options.table, options.column) - if (state?.phase !== 'cut-over') { + if (state?.phase !== requiredPhase) { p.log.error( - `Cannot generate drop migration: ${options.table}.${options.column} is in phase '${state?.phase ?? '—'}'. Must be 'cut-over'.`, + `Cannot generate drop migration: ${options.table}.${options.column} is in phase '${state?.phase ?? '—'}'. Must be '${requiredPhase}'${isV3 ? ' (EQL v3 has no cut-over — backfill, switch the app to the encrypted column, then drop)' : ''}.`, ) exitCode = 1 return } + if (isV3) { + p.log.info( + `${options.table}.${options.column}_encrypted is EQL v3 — the drop targets the original plaintext column "${options.column}" (v3 has no rename, so there is no "${options.column}_plaintext"). Make sure your application reads/writes ${options.column}_encrypted before applying this migration.`, + ) + } + const dot = options.table.indexOf('.') const qualifiedTable = dot >= 0 ? `"${options.table.slice(0, dot)}"."${options.table.slice(dot + 1)}"` : `"${options.table}"` - const dropSql = `-- Generated by stash encrypt drop\n-- Drops the plaintext column now that ${options.table}.${options.column} is encrypted.\n\nALTER TABLE ${qualifiedTable} DROP COLUMN "${options.column}_plaintext";\n` + const dropSql = `-- Generated by stash encrypt drop\n-- Drops the plaintext column now that ${options.table}.${options.column}${isV3 ? '_encrypted' : ''} is encrypted.\n\nALTER TABLE ${qualifiedTable} DROP COLUMN "${plaintextToDrop}";\n` const cwd = process.cwd() const migrationsDir = options.migrationsDir ?? 'drizzle' @@ -85,7 +108,7 @@ export async function dropCommand(options: DropCommandOptions) { // Without this, the file ships but `drizzle-kit migrate` never picks it // up because the journal doesn't reference it. const result = await scaffoldDrizzleMigration({ - name: `drop_${options.table}_${options.column}_plaintext`, + name: `drop_${options.table}_${plaintextToDrop}`, outDir: migrationsDir, sql: dropSql, }) @@ -100,7 +123,7 @@ export async function dropCommand(options: DropCommandOptions) { .toISOString() .replace(/[-:.TZ]/g, '') .slice(0, 14) - const fileName = `${ts}_drop_${options.table}_${options.column}_plaintext.sql` + const fileName = `${ts}_drop_${options.table}_${plaintextToDrop}.sql` filePath = path.join(dirAbs, fileName) fs.writeFileSync(filePath, dropSql, 'utf-8') nextStep = `Review the migration, then apply with your migration tool:\n - prisma migrate deploy\n - psql -f ${fileName}` @@ -129,6 +152,11 @@ export async function dropCommand(options: DropCommandOptions) { exitCode = 1 } finally { await client.end() + // In `finally` (not after the try/catch) deliberately: the precondition + // guards above `return` from inside `try`, which skips any code placed + // after the block — so a trailing `if (exitCode) process.exit(...)` + // was unreachable on exactly the failure paths it existed for, and + // guard failures exited 0. + if (exitCode) process.exit(exitCode) } - if (exitCode) process.exit(exitCode) } diff --git a/packages/cli/src/commands/encrypt/status.ts b/packages/cli/src/commands/encrypt/status.ts index 8591fbbbb..5aeeb0a34 100644 --- a/packages/cli/src/commands/encrypt/status.ts +++ b/packages/cli/src/commands/encrypt/status.ts @@ -65,6 +65,7 @@ export async function statusCommand() { state: stateMap.get(key) ?? null, eqlColumn: eqlConfig.get(key) ?? null, physicalColumns: physicalCols.get(tableName) ?? new Set(), + eqlVersion: column.eqlVersion, }), ) } @@ -116,6 +117,10 @@ function renderRow(input: { tableName: string columnName: string intentIndexes: string[] | undefined + /** From the manifest (recorded at backfill time). v3 columns have no + * `eql_v2_configuration` row and no rename, so the v2 drift flags don't + * apply. Absent = written by pre-v3 tooling = v2. */ + eqlVersion?: 2 | 3 state: { phase: MigrationPhase rowsProcessed: number | null @@ -134,7 +139,9 @@ function renderRow(input: { } = input const phase = state?.phase ?? (intentIndexes ? 'schema-added' : '—') - const eql = eqlColumn ? eqlColumn.state : '—' + // v3 columns have no eql_v2_configuration row by design — show the + // version rather than a misleading blank. + const eql = eqlColumn ? eqlColumn.state : input.eqlVersion === 3 ? 'v3' : '—' const indexes = eqlColumn ? eqlColumn.indexes.join(', ') || '(none)' : intentIndexes?.join(', ') || '—' @@ -147,12 +154,20 @@ function renderRow(input: { // writes covered every row from seeding). Frame per phase. const progress = formatProgress(phase, state) + const isV3 = input.eqlVersion === 3 const flags: string[] = [] - if (intentIndexes && !eqlColumn) flags.push('not-registered') + // v2-only drift flags: a v3 column is never registered in + // `eql_v2_configuration` (no config table exists) and never reaches the + // rename that creates `_plaintext`. + if (!isV3 && intentIndexes && !eqlColumn) flags.push('not-registered') if (intentIndexes && !physicalColumns.has(`${columnName}_encrypted`)) { flags.push('encrypted-col-missing') } - if (phase === 'cut-over' && !physicalColumns.has(`${columnName}_plaintext`)) { + if ( + !isV3 && + phase === 'cut-over' && + !physicalColumns.has(`${columnName}_plaintext`) + ) { flags.push('plaintext-col-missing') } diff --git a/packages/migrate/README.md b/packages/migrate/README.md index a4fa2959e..8456603b5 100644 --- a/packages/migrate/README.md +++ b/packages/migrate/README.md @@ -1,20 +1,22 @@ # @cipherstash/migrate -Primitives for migrating existing plaintext columns to CipherStash's `eql_v2_encrypted` in production Postgres databases, safely and resumably. +Primitives for migrating existing plaintext columns to CipherStash encrypted columns (EQL v2 `eql_v2_encrypted` or the concrete EQL v3 `eql_v3_*` domains) in production Postgres databases, safely and resumably. Backs the `stash encrypt` CLI command group, but also exported for direct use — embed `runBackfill()` in your own worker or cron job when you'd rather not pipe gigabytes through a CLI process. ## Lifecycle -Each column walks through these phases: +Each column walks through these phases — the ladder depends on the column's EQL version (auto-detected from its Postgres domain type via `detectColumnEqlVersion`): ``` -schema-added → dual-writing → backfilling → backfilled → cut-over → dropped +EQL v2: schema-added → dual-writing → backfilling → backfilled → cut-over → dropped +EQL v3: schema-added → dual-writing → backfilling → backfilled ————————————→ dropped ``` -State is tracked in an append-only `cipherstash.cs_migrations` table installed by `stash eql install`. The EQL intent (which indexes, which cast_as) continues to live in `eql_v2_configuration` so Proxy continues to work against the same database. +State is tracked in an append-only `cipherstash.cs_migrations` table installed by `stash eql install`. -> **This package targets EQL v2.** The lifecycle above wraps `eql_v2.*` functions and reads and writes `eql_v2_configuration`. `stash eql install` installs v2 by default; EQL v3 is opt-in via `stash eql install --eql-version 3`, installs via the direct path only, and encodes its configuration in each column's domain type rather than in a configuration table. Columns installed under v3 are not covered by this package's migration lifecycle. +- **EQL v2** additionally keeps its intent (indexes, cast_as) in `eql_v2_configuration` so CipherStash Proxy works against the same database, and finishes with a **cut-over**: `eql_v2.rename_encrypted_columns()` swaps `_encrypted` into place (`` becomes `_plaintext`) alongside a config promotion. +- **EQL v3** has **no configuration table and no cut-over** — each column's domain type encodes its own configuration. The application switches to `_encrypted` *by name*, and the original plaintext `` is dropped once verified. Backfill verification is a plain count of the populated target column (`countEncrypted`); the concrete `eql_v3_*` domain's CHECK constraint guarantees every non-null value is a valid v3 envelope. ## API @@ -41,7 +43,7 @@ Creates `cipherstash.cs_migrations` idempotently. Normally called by `stash eql Chunked, resumable, idempotent backfill of plaintext → encrypted. Per chunk, in a single transaction: select next page → encrypt via `client.bulkEncryptModels` → `UPDATE … FROM (VALUES …)` → `INSERT` a `backfill_checkpoint` event. Guards with `encrypted IS NULL` so re-runs never double-write. - `db`: a `pg.PoolClient` (the runner drives transactions on it). -- `encryptionClient`: your initialised `@cipherstash/stack` `EncryptionClient` (or anything that exposes `bulkEncryptModels(models, table)` returning `{ data } | { failure }`). +- `encryptionClient`: your initialised `@cipherstash/stack` client (or anything that exposes `bulkEncryptModels(models, table)` returning `{ data } | { failure }`). For an EQL v3 column pass an `EncryptionV3` client (from `@cipherstash/stack/v3`) — it pins the v3 wire format; the engine itself is version-agnostic and writes whatever envelope the client produces. - `tableSchema`: the `EncryptedTable` for the target table from your encryption client file. - `signal`: optional `AbortSignal`. If aborted between chunks, the backfill exits cleanly and leaves a resumable checkpoint. @@ -53,7 +55,11 @@ Direct access to the `cs_migrations` event log. Use these if you're building you ### `renameEncryptedColumns(client)` / `reloadConfig(client)` -Thin wrappers around `eql_v2.rename_encrypted_columns()` (the cut-over primitive) and `eql_v2.reload_config()` (Proxy refresh hint — no-op when connected directly to Postgres). +Thin wrappers around `eql_v2.rename_encrypted_columns()` (the **v2** cut-over primitive) and `eql_v2.reload_config()` (Proxy refresh hint — no-op when connected directly to Postgres). Not used in the v3 lifecycle — v3 has no rename step. + +### `detectColumnEqlVersion(client, table, column)` / `countEncrypted(client, table, column)` + +`detectColumnEqlVersion` inspects the column's Postgres domain type and returns `'v2'`, `'v3'`, or `null` (not an EQL column) — the branch point for everything version-specific above. `countEncrypted` counts populated target-column rows: the v3 backfill-verification primitive (v2 verified through `eql_v2.count_encrypted_with_active_config`, which needs the config table v3 doesn't have). ### `readManifest(cwd)` / `writeManifest(manifest, cwd)` diff --git a/packages/migrate/src/__tests__/backfill-v3.integration.test.ts b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts new file mode 100644 index 000000000..91e5c58ec --- /dev/null +++ b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts @@ -0,0 +1,192 @@ +/** + * EQL v3 twin of `backfill.integration.test.ts` — the engine mechanics are + * identical (and exhaustively covered there); THIS file pins the v3-specific + * contract (#648/#649): + * + * - the leak guard (`isEncryptedPayload`) accepts BOTH v3 wire shapes — flat + * scalars (`{v:3, i, c}`) and SteVec documents (`{v:3, k:'sv', i, sv}`) — + * so a v3 client's output flows through `runBackfill` unmodified; + * - the `$N::jsonb` write lands v3 envelopes in the target column; + * - `countEncrypted` (the v3 verification primitive — v3 has no + * `eql_v2.count_encrypted_with_active_config`) counts them. + * + * Skipped unless `PG_TEST_URL` is set (same harness as the v2 file): + * + * ``` + * cd local && docker compose up -d + * PG_TEST_URL=postgres://cipherstash:password@localhost:5432/cipherstash \ + * pnpm -F @cipherstash/migrate test backfill-v3.integration + * ``` + * + * No CipherStash credentials required — payloads are deterministic v3-shaped + * markers. End-to-end proof against a real `eql_v3_*` domain (whose CHECK + * constraint demands real ciphertext structure) lives with the live-crypto + * harness, not here. + */ + +import 'dotenv/config' +import pg from 'pg' +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest' +import { type EncryptionClientLike, runBackfill } from '../backfill.js' +import { countEncrypted } from '../cursor.js' +import { installMigrationsSchema } from '../install.js' +import { progress } from '../state.js' + +const PG_URL = process.env.PG_TEST_URL +const runIntegration = Boolean(PG_URL) + +describe.skipIf(!runIntegration)('runBackfill with EQL v3 payloads', () => { + let pool: pg.Pool + + beforeAll(async () => { + pool = new pg.Pool({ connectionString: PG_URL, max: 4 }) + const db = await pool.connect() + try { + await db.query('DROP SCHEMA IF EXISTS cipherstash CASCADE') + await db.query('DROP SCHEMA IF EXISTS migrate_v3_test CASCADE') + await db.query('CREATE SCHEMA migrate_v3_test') + await installMigrationsSchema(db) + } finally { + db.release() + } + }) + + afterEach(async () => { + await pool.query('DROP TABLE IF EXISTS migrate_v3_test.users') + await pool.query('TRUNCATE cipherstash.cs_migrations') + }) + + afterAll(async () => { + await pool.end() + }) + + /** v3 FLAT SCALAR marker — the shape `EncryptionV3` clients emit for + * text/number columns: no `k` discriminator, top-level `c`. */ + const v3ScalarClient: EncryptionClientLike = { + bulkEncryptModels(input) { + return Promise.resolve({ + data: input.map((row) => ({ + __pk: row.__pk, + email: { + v: 3, + i: { t: 'users', c: 'email' }, + c: `mock-v3-ciphertext:${row.email}`, + }, + })), + }) + }, + } + + /** v3 SteVec marker — the searchable-JSON document shape: `k: 'sv'`, + * `sv` array, NO top-level `c`. The leak guard must accept it. */ + const v3SteVecClient: EncryptionClientLike = { + bulkEncryptModels(input) { + return Promise.resolve({ + data: input.map((row) => ({ + __pk: row.__pk, + email: { + v: 3, + k: 'sv', + i: { t: 'users', c: 'email' }, + sv: [{ s: 'mock-selector', t: `mock-term:${row.email}` }], + }, + })), + }) + }, + } + + async function seed(n: number) { + const db = await pool.connect() + try { + await db.query(` + CREATE TABLE migrate_v3_test.users ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + email text NOT NULL, + email_encrypted jsonb + ) + `) + await db.query( + `INSERT INTO migrate_v3_test.users (email) + SELECT 'user-' || g || '@example.com' FROM generate_series(1, $1) AS g`, + [n], + ) + } finally { + db.release() + } + } + + function backfillWith(client: EncryptionClientLike) { + return (db: pg.PoolClient) => + runBackfill({ + db, + encryptionClient: client, + tableSchema: { tableName: 'users', build: () => ({}) }, + tableName: 'migrate_v3_test.users', + schemaColumnKey: 'email', + plaintextColumn: 'email', + encryptedColumn: 'email_encrypted', + pkColumn: 'id', + chunkSize: 50, + }) + } + + it('backfills v3 flat-scalar payloads end to end', async () => { + await seed(120) + const db = await pool.connect() + try { + const result = await backfillWith(v3ScalarClient)(db) + expect(result.completed).toBe(true) + expect(result.rowsProcessed).toBe(120) + + // countEncrypted is the v3 verification primitive (no config table). + expect( + await countEncrypted(db, 'migrate_v3_test.users', 'email_encrypted'), + ).toBe(120) + + // The stored value is the v3 envelope, verbatim. + const row = await db.query<{ enc: { v: number; c: string } }>( + 'SELECT email_encrypted AS enc FROM migrate_v3_test.users ORDER BY id LIMIT 1', + ) + expect(row.rows[0]?.enc.v).toBe(3) + expect(row.rows[0]?.enc.c).toContain('mock-v3-ciphertext:') + + const state = await progress(db, 'migrate_v3_test.users', 'email') + expect(state?.phase).toBe('backfilled') + } finally { + db.release() + } + }) + + it('the leak guard accepts v3 SteVec documents (sv, no top-level c)', async () => { + await seed(30) + const db = await pool.connect() + try { + const result = await backfillWith(v3SteVecClient)(db) + expect(result.completed).toBe(true) + expect(result.rowsProcessed).toBe(30) + const row = await db.query<{ enc: { v: number; k: string } }>( + 'SELECT email_encrypted AS enc FROM migrate_v3_test.users ORDER BY id LIMIT 1', + ) + expect(row.rows[0]?.enc.v).toBe(3) + expect(row.rows[0]?.enc.k).toBe('sv') + } finally { + db.release() + } + }) + + it('is idempotent for v3 exactly like v2 (encrypted IS NULL guard)', async () => { + await seed(40) + const db = await pool.connect() + try { + await backfillWith(v3ScalarClient)(db) + const second = await backfillWith(v3ScalarClient)(db) + expect(second.completed).toBe(true) + expect(second.rowsProcessed).toBe(0) + expect( + await countEncrypted(db, 'migrate_v3_test.users', 'email_encrypted'), + ).toBe(40) + } finally { + db.release() + } + }) +}) diff --git a/packages/migrate/src/cursor.ts b/packages/migrate/src/cursor.ts index f79e5aeef..4546daf4f 100644 --- a/packages/migrate/src/cursor.ts +++ b/packages/migrate/src/cursor.ts @@ -120,6 +120,29 @@ export async function countUnencrypted( return Number(result.rows[0]?.count ?? 0) } +/** + * Count rows whose encrypted column is populated: `encrypted IS NOT NULL`. + * + * The v3 backfill-verification primitive. EQL v2 verified via + * `eql_v2.count_encrypted_with_active_config(...)`, which reads the + * `eql_v2_configuration` table — v3 has no configuration table, so the + * equivalent check is a plain count of the target column (the concrete + * `eql_v3_*` domain's CHECK constraint already guarantees every non-null + * value is a valid v3 envelope). + */ +export async function countEncrypted( + client: ClientBase, + tableName: string, + encryptedColumn: string, +): Promise { + const enc = quoteIdent(encryptedColumn) + const table = qualifyTable(tableName) + const result = await client.query<{ count: string }>( + `SELECT count(*)::text AS count FROM ${table} WHERE ${enc} IS NOT NULL`, + ) + return Number(result.rows[0]?.count ?? 0) +} + /** * Quote a possibly schema-qualified table name for use in a SQL statement. * `"foo"` → `"foo"`; `"public.foo"` → `"public"."foo"`. Use for identifiers diff --git a/packages/migrate/src/index.ts b/packages/migrate/src/index.ts index dfaec9e65..79c400aa4 100644 --- a/packages/migrate/src/index.ts +++ b/packages/migrate/src/index.ts @@ -31,6 +31,7 @@ export { runBackfill, } from './backfill.js' export { + countEncrypted, countUnencrypted, fetchUnencryptedPage, type KeysetPage, diff --git a/packages/migrate/src/manifest.ts b/packages/migrate/src/manifest.ts index c3ad4043e..943b2f463 100644 --- a/packages/migrate/src/manifest.ts +++ b/packages/migrate/src/manifest.ts @@ -60,6 +60,15 @@ const ManifestColumnSchema = z.object({ * CLI auto-detect via `information_schema`. */ pkColumn: z.string().optional(), + /** + * The EQL version of the encrypted target column, recorded at backfill + * time from `detectColumnEqlVersion`. Lets `encrypt status`/`plan` + * branch without a DB round-trip: v3 columns have no + * `eql_v2_configuration` registration and no rename cut-over, so the + * v2-specific drift flags don't apply. Absent on manifests written + * before v3 support — those columns are necessarily v2. + */ + eqlVersion: z.union([z.literal(2), z.literal(3)]).optional(), }) /** diff --git a/packages/migrate/vitest.config.ts b/packages/migrate/vitest.config.ts new file mode 100644 index 000000000..e0661f536 --- /dev/null +++ b/packages/migrate/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + // The two backfill integration suites (v2 + v3) share one real database + // and the singleton `cipherstash.cs_migrations` schema — run in parallel + // they race on schema DROP/CREATE and truncate each other's state rows. + // Everything here is fast enough that whole-package serialization is + // cheaper than a locking scheme. + fileParallelism: false, + }, +}) diff --git a/skills/stash-cli/SKILL.md b/skills/stash-cli/SKILL.md index 516d9c6e8..ffb1342ce 100644 --- a/skills/stash-cli/SKILL.md +++ b/skills/stash-cli/SKILL.md @@ -439,6 +439,8 @@ stash encrypt backfill --table users --column email --chunk-size 5000 Chunked, resumable, idempotent. Walks the table in keyset-pagination order, encrypts each chunk via `bulkEncryptModels`, and writes one `UPDATE ... FROM (VALUES ...)` per chunk in a transaction that also checkpoints to `cs_migrations`. SIGINT/SIGTERM finishes the current chunk and exits cleanly; re-running resumes. The ` IS NOT NULL AND _encrypted IS NULL` guard makes concurrent runners and re-runs converge. +Backfill **auto-detects the target column's EQL version** from its Postgres domain type and records it (plus the version-appropriate target phase) in `.cipherstash/migrations.json`. On an EQL v3 column it finishes by printing the v3 next steps: switch the application to `_encrypted` by name, then `stash encrypt drop` — there is no cut-over. + **Dual-write precondition.** The application must already write both `` and `_encrypted` on every insert and update. Otherwise rows written *during* the backfill land in plaintext only, silently. The first run prompts (interactive) or requires `--confirm-dual-writes-deployed` (non-interactive), then records `dual_writing`. Resumes don't re-prompt. | Flag | Description | @@ -457,13 +459,13 @@ Chunked, resumable, idempotent. Walks the table in keyset-pagination order, encr stash encrypt cutover --table users --column email ``` -**Preconditions:** the column is in the `backfilled` phase, **and** a pending EQL configuration exists. +**EQL v2 only** — v3 has no cut-over: the application switches to `_encrypted` by name, and running this command on a v3 column reports "not applicable" (exit 0) with the next step. For v2, the preconditions are: the column is in the `backfilled` phase, **and** a pending EQL configuration exists. In one transaction it renames `` → `_plaintext` and `_encrypted` → ``, advances the pending config to `encrypting`, activates it, and appends a `cut_over` event. With a Proxy URL configured (`--proxy-url` or `CIPHERSTASH_PROXY_URL`) it then calls `eql_v2.reload_config()` so Proxy picks up the new shape. > **After cutover, `` holds ciphertext — the read path is not automatic.** Wire reads through the encryption client (`decryptModel(row, table)` for Drizzle, the `encryptedSupabaseV3` wrapper for Supabase — `encryptedSupabase` on the legacy v2 surface, otherwise `decrypt` / `bulkDecryptModels`) before returning values to callers. Skip this and your read paths hand raw EQL payloads to end users. The integration skill has the exact API. **CipherStash Proxy is the one exception** — it decrypts on the wire, so Proxy users need no application change. The cutover plan written by `stash plan` includes this read-path switch as an explicit step. -> **Known gap.** The pending-configuration precondition is satisfied by `stash db push`. SDK-only users (who otherwise never need `db push`) must therefore run it once before `encrypt cutover`. Decoupling this — under EQL v3 there is no configuration table at all — is tracked in [issue #585](https://github.com/cipherstash/stack/issues/585). +> **Known gap (v2).** The pending-configuration precondition is satisfied by `stash db push`. SDK-only users (who otherwise never need `db push`) must therefore run it once before `encrypt cutover`. (EQL v3 columns sidestep this entirely — no configuration table, no cutover; see above.) Tracked in [issue #585](https://github.com/cipherstash/stack/issues/585). Flags: `--table`, `--column`, `--proxy-url `, `--migrations-dir `. @@ -473,7 +475,7 @@ Flags: `--table`, `--column`, `--proxy-url `, `--migrations-dir `. stash encrypt drop --table users --column email ``` -For columns in the `cut_over` phase. Detects your migration tooling and emits a migration containing `ALTER TABLE DROP COLUMN _plaintext;`. It does **not** apply it — review and run your own migrate command. The `dropped` event is recorded only after a later `encrypt status` confirms the column is gone. +Version-aware. For **EQL v2** columns in the `cut-over` phase it emits `ALTER TABLE
DROP COLUMN _plaintext;` (the post-rename name). For **EQL v3** columns it runs from the `backfilled` phase and drops the ORIGINAL `` — v3 has no rename, so make sure the application reads/writes `_encrypted` first. Either way it does **not** apply the migration — review and run your own migrate command. Flags: `--table`, `--column`, `--migrations-dir `. diff --git a/skills/stash-encryption/SKILL.md b/skills/stash-encryption/SKILL.md index 73a7a88ff..a31e9f455 100644 --- a/skills/stash-encryption/SKILL.md +++ b/skills/stash-encryption/SKILL.md @@ -728,7 +728,7 @@ try { ## Rolling Encryption Out to Production -> **EQL v2 note ([#648](https://github.com/cipherstash/stack/issues/648)).** The backfill/cutover tooling in this section (`stash encrypt *`, `stash db push`, `@cipherstash/migrate`) currently targets **EQL v2 columns** — the `eql_v2_configuration` registry, `eql_v2_encrypted` payload columns, and v2 schemas. v3 support for the rollout lifecycle is tracked in [cipherstash/stack#648](https://github.com/cipherstash/stack/issues/648). The *process* below — rollout, deploy gate, cutover — is the model to follow either way; every `eql_v2`-named artifact it mentions is part of that v2-targeting tooling. +> **EQL version note.** The rollout tooling (`stash encrypt *`, `@cipherstash/migrate`) works with **both EQL versions** and auto-detects the column's version from its Postgres domain type — no flag. The lifecycles differ at the end: **v2** finishes with `stash encrypt cutover` (a rename swap plus a config promotion in `eql_v2_configuration`), then drops `_plaintext`. **v3 has no cut-over and no configuration table** — after backfill you point the application at `_encrypted` *by name*, verify reads, then `stash encrypt drop` generates the drop of the original plaintext ``. Running `encrypt cutover` on a v3 column safely reports "not applicable" with the next step. `stash db push`/`db activate` remain v2-only (they manage `eql_v2_configuration`). Adding a fresh encrypted column to a table you don't yet write to is the easy case — declare it in the schema, run the migration, start writing. The harder case is taking an **existing plaintext column with live data** and turning it into an encrypted one without dropping a write or returning the wrong value mid-cutover. @@ -912,7 +912,7 @@ await runBackfill({ }) ``` -Useful when the backfill needs to run in a worker, on a schedule, or alongside an existing job runner. (Like the CLI, `runBackfill` currently targets EQL v2 columns — see the note at the top of this section.) +Useful when the backfill needs to run in a worker, on a schedule, or alongside an existing job runner. `runBackfill` is version-agnostic — for an EQL v3 column pass an `EncryptionV3` client (from `@cipherstash/stack/v3`) and it writes v3 envelopes straight into the concrete `eql_v3_*` domain column. ### Invariants the rollout preserves From 8b8a7127840dfca505d0af0ae5cdbb603c3df194 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Thu, 16 Jul 2026 22:31:13 +1000 Subject: [PATCH 3/6] fix(migrate,cli): resolve EQL columns from domain types; verify before drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code-review round on #649. The EQL v3 types are self-describing — that was the point of v3 — so version and encrypted-column resolution now come from the Postgres domain types, with the _encrypted naming demoted to an unenforced convention: - @cipherstash/migrate: classifyEqlDomain (the one home for the rule), listEncryptedColumns, resolveEncryptedColumn (hint → convention → sole-EQL-column). detectColumnEqlVersion resolves tables case-exactly (format('%I') before to_regclass) — a bare to_regclass case-folded Prisma-style "User" tables to 'user' and silently wedged v3 columns into the v2 lifecycle. EqlVersion is numeric (2|3), matching the manifest and installer; no more string↔number translation. - The manifest records encryptedColumn at backfill time; cutover and drop resolve manifest-hint-first instead of guessing the name, so a custom --encrypted-column no longer deadlocks the v3 lifecycle. - encrypt drop (v3) now verifies live coverage before generating the migration: refuses while any row has plaintext set and ciphertext NULL (countUnencrypted) — the verification the README promised but nothing performed. Dropping the original column is the irreversible step; the phase gate alone only proved a backfill once finished. - encrypt cutover (v3) is phase-aware: mid-backfill it exits 1 telling the user to finish the backfill, instead of exit-0 guidance to switch the app onto a half-populated column. The v2 path guards the eql_v2_configuration query so v3-only databases get an explanation, not a raw relation-does-not-exist error. - encrypt status classifies from the observed domain type (manifest as fallback) — a v3 column whose manifest entry predates detection no longer collects permanent false not-registered flags. stash status's quest ladder and the init agent handoff teach the v3 next step (4-rung ladder, switch-by-name) instead of steering v3 users into the no-op cutover. - Docs trued up: manifest docstrings, migrate README (drop's built-in coverage gate), skills' phase-ladder intros (both lifecycles), and the stale workflow comments. Unit + real-catalog integration coverage for the resolver (mixed-case tables, custom names, ambiguity); CLI tests pin the coverage gate, the phase-aware cutover, and resolved-name usage. 507 CLI + 49 migrate + 58 pty-e2e green. Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .changeset/migrate-eql-v3.md | 50 +++-- .../encrypt/__tests__/encrypt-v3.test.ts | 188 ++++++++++++++++-- packages/cli/src/commands/encrypt/backfill.ts | 26 ++- packages/cli/src/commands/encrypt/cutover.ts | 47 ++++- packages/cli/src/commands/encrypt/drop.ts | 43 +++- .../src/commands/encrypt/lib/db-readers.ts | 32 ++- .../src/commands/encrypt/lib/resolve-eql.ts | 58 ++++++ packages/cli/src/commands/encrypt/status.ts | 42 +++- .../init/lib/__tests__/setup-prompt.test.ts | 10 +- .../cli/src/commands/init/lib/setup-prompt.ts | 12 +- packages/cli/src/commands/status/index.ts | 15 +- packages/cli/src/commands/status/quest.ts | 47 ++++- packages/migrate/README.md | 10 +- .../__tests__/backfill-v3.integration.test.ts | 89 +++++++++ .../migrate/src/__tests__/version.test.ts | 181 ++++++++++++----- packages/migrate/src/index.ts | 18 +- packages/migrate/src/manifest.ts | 20 +- packages/migrate/src/version.ts | 157 +++++++++++++-- skills/stash-cli/SKILL.md | 9 +- skills/stash-encryption/SKILL.md | 4 +- 20 files changed, 875 insertions(+), 183 deletions(-) create mode 100644 packages/cli/src/commands/encrypt/lib/resolve-eql.ts diff --git a/.changeset/migrate-eql-v3.md b/.changeset/migrate-eql-v3.md index 08ef82026..f504169b3 100644 --- a/.changeset/migrate-eql-v3.md +++ b/.changeset/migrate-eql-v3.md @@ -4,30 +4,46 @@ --- EQL v3 support for the encryption rollout lifecycle (#648). The `stash -encrypt *` commands (and `@cipherstash/migrate` underneath) now auto-detect a -column's EQL version from its Postgres domain type and follow the right -lifecycle — no new flags: +encrypt *` commands (and `@cipherstash/migrate` underneath) now resolve a +column's EQL version and its encrypted counterpart from the **Postgres domain +types** — the EQL v3 types are self-describing, so the `_encrypted` +naming is a convention only, never enforced or relied upon — and follow the +right lifecycle, no new flags: - **`encrypt backfill`** works on v3 columns unchanged (the engine was always version-agnostic; pass an `EncryptionV3` client and real v3 envelopes land in the concrete `eql_v3_*` domain column — verified live against a real database, including the domain CHECK and a decrypt round-trip). The - manifest records the detected version and the v3 target phase, and the - command prints v3-appropriate next steps. -- **`encrypt cutover`** on a v3 column reports "not applicable" (exit 0) with - guidance: v3 has no rename cut-over — the application switches to - `_encrypted` by name. -- **`encrypt drop`** is version-aware: v3 runs from the `backfilled` phase - and drops the ORIGINAL plaintext column (there is no `_plaintext` - under v3); v2 behaviour is unchanged. -- **`encrypt status`** no longer raises the v2-only `not-registered` / - `plaintext-col-missing` drift flags for v3 columns (v3 has no - `eql_v2_configuration` and no rename) and shows `v3` in the EQL column. -- New `@cipherstash/migrate` exports: `countEncrypted` (the v3 - backfill-verification primitive) and a manifest `eqlVersion` field. + manifest records the detected version, the encrypted column's name, and the + v3 target phase, and the command prints v3-appropriate next steps. +- **`encrypt cutover`** on a backfilled v3 column reports "not applicable" + (exit 0) with guidance: v3 has no rename cut-over — the application + switches to the encrypted column by name. Before backfill completes it + exits 1 and says to finish the backfill instead of instructing the switch. + On a database with no `eql_v2_configuration` table (a v3-only install) the + v2 path now explains that instead of surfacing a raw Postgres error. +- **`encrypt drop`** is version-aware: v3 runs from the `backfilled` phase, + **verifies live coverage** (refuses to generate the migration while any row + still has the plaintext set and the encrypted column NULL — the + `countUnencrypted` check), and drops the ORIGINAL plaintext column (there + is no `_plaintext` under v3); v2 behaviour is unchanged. +- **`encrypt status`** classifies each column from the observed domain type + (manifest as fallback), shows `v3` in the EQL column, and no longer raises + the v2-only `not-registered` / `plaintext-col-missing` drift flags for v3 + columns. `stash status`'s quest ladder and the `stash init` agent handoff + prompt teach the version-appropriate next step (no more "run cutover" on + v3 columns). +- New `@cipherstash/migrate` exports: `classifyEqlDomain`, + `resolveEncryptedColumn`, `listEncryptedColumns` (domain-type resolution — + case-exact for quoted/mixed-case table names), `countEncrypted` / + `countUnencrypted` (coverage counts), and manifest `eqlVersion` + + `encryptedColumn` fields. `EqlVersion` is numeric (`2 | 3`), matching the + manifest and the installer. - Fixed: `encrypt cutover`/`encrypt drop` precondition failures now actually exit 1 — the early-return guards previously skipped the exit-code path - entirely, so failed preconditions exited 0. + entirely, so failed preconditions exited 0. (This also applies to v2 + preconditions: scripted pipelines that relied on the erroneous exit 0 will + now see the documented exit 1.) The `stash-cli` and `stash-encryption` skills and the `@cipherstash/migrate` README document the two lifecycles (v2: backfill → cutover → drop; diff --git a/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts b/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts index 9e48bfc0a..b8e7c8eb5 100644 --- a/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts +++ b/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts @@ -3,10 +3,15 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' // The EQL-v3 lifecycle branches (#648/#649): v3 has no cut-over rename — // `encrypt cutover` must short-circuit with guidance instead of running the // v2 config machine, and `encrypt drop` must target the ORIGINAL plaintext -// column (there is no `_plaintext`) gated on `backfilled` rather than -// `cut-over`. The v2 paths are pinned alongside as regression guards. +// column (there is no `_plaintext`) gated on `backfilled` AND a live +// coverage check. Version + encrypted-column NAME come from the domain types +// via `resolveEncryptedColumn` — the `_encrypted` naming is a +// convention, never relied upon. The v2 paths are pinned alongside as +// regression guards. -const queryMock = vi.hoisted(() => vi.fn(async () => ({ rows: [] }))) +const queryMock = vi.hoisted(() => + vi.fn(async (_sql: string) => ({ rows: [] as unknown[] })), +) vi.mock('pg', () => ({ default: { Client: class { @@ -17,8 +22,19 @@ vi.mock('pg', () => ({ }, })) +type ColumnInfo = { column: string; domain: string; version: 2 | 3 } + const migrateMocks = vi.hoisted(() => ({ - detectColumnEqlVersion: vi.fn(async () => 'v3' as 'v2' | 'v3' | null), + resolveEncryptedColumn: vi.fn( + async (): Promise => ({ + column: 'email_encrypted', + domain: 'eql_v3_text_search', + version: 3, + }), + ), + listEncryptedColumns: vi.fn(async (): Promise => []), + readManifest: vi.fn(async () => null), + countUnencrypted: vi.fn(async () => 0), progress: vi.fn( async () => ({ phase: 'backfilled' }) as { phase: string } | null, ), @@ -72,14 +88,52 @@ import * as p from '@clack/prompts' import { cutoverCommand } from '../cutover.js' import { dropCommand } from '../drop.js' +const V3_INFO: ColumnInfo = { + column: 'email_encrypted', + domain: 'eql_v3_text_search', + version: 3, +} +const V3_CUSTOM_INFO: ColumnInfo = { + column: 'email_enc', + domain: 'eql_v3_text_search', + version: 3, +} +const V2_INFO: ColumnInfo = { + column: 'email_encrypted', + domain: 'eql_v2_encrypted', + version: 2, +} + +/** v2 config machine present + a pending config row. */ +function mockV2ConfigQueries() { + queryMock.mockImplementation(async (sql: string) => { + if (typeof sql !== 'string') return { rows: [] } + if (sql.includes('to_regclass')) { + return { rows: [{ exists: 'eql_v2_configuration' }] } + } + if (sql.includes('eql_v2_configuration')) { + return { rows: [{ exists: true }] } + } + return { rows: [] } + }) +} + +function spyExit() { + return vi + .spyOn(process, 'exit') + .mockImplementation((() => undefined) as never) +} + describe('encrypt cutover — EQL version awareness', () => { beforeEach(() => { vi.clearAllMocks() + queryMock.mockResolvedValue({ rows: [] }) migrateMocks.progress.mockResolvedValue({ phase: 'backfilled' }) + migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_INFO) }) - it('short-circuits on a v3 column: guidance, no rename, no config machine', async () => { - migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v3') + it('short-circuits on a backfilled v3 column: guidance, no rename, no config machine, exit 0', async () => { + const exitSpy = spyExit() await cutoverCommand({ table: 'users', column: 'email' }) @@ -93,16 +147,44 @@ describe('encrypt cutover — EQL version awareness', () => { expect(migrateMocks.migrateConfig).not.toHaveBeenCalled() expect(migrateMocks.activateConfig).not.toHaveBeenCalled() expect(migrateMocks.appendEvent).not.toHaveBeenCalled() + expect(exitSpy).not.toHaveBeenCalled() + exitSpy.mockRestore() }) - it('still runs the v2 flow for a v2 column (regression pin)', async () => { - migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v2') - // pending-config check returns true - queryMock.mockImplementation(async (sql: string) => - typeof sql === 'string' && sql.includes('eql_v2_configuration') - ? { rows: [{ exists: true }] } - : { rows: [] }, + it('v3 mid-backfill: does NOT tell the user to switch; exits 1', async () => { + migrateMocks.progress.mockResolvedValue({ phase: 'dual-writing' }) + const exitSpy = spyExit() + + await cutoverCommand({ table: 'users', column: 'email' }) + + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining("hasn't finished backfilling"), + ) + // The switch-now guidance must not appear before backfill completes — + // following it mid-backfill reads NULLs for unbackfilled rows. + expect(p.log.info).not.toHaveBeenCalledWith( + expect.stringContaining('point your application'), + ) + expect(exitSpy).toHaveBeenCalledWith(1) + exitSpy.mockRestore() + }) + + it('uses the RESOLVED encrypted column name, not the naming convention', async () => { + migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_CUSTOM_INFO) + + await cutoverCommand({ table: 'users', column: 'email' }) + + expect(p.log.info).toHaveBeenCalledWith( + expect.stringContaining('email_enc'), + ) + expect(p.log.info).not.toHaveBeenCalledWith( + expect.stringContaining('email_encrypted'), ) + }) + + it('still runs the v2 flow for a v2 column (regression pin)', async () => { + migrateMocks.resolveEncryptedColumn.mockResolvedValue(V2_INFO) + mockV2ConfigQueries() await cutoverCommand({ table: 'users', column: 'email' }) @@ -110,19 +192,50 @@ describe('encrypt cutover — EQL version awareness', () => { expect(migrateMocks.migrateConfig).toHaveBeenCalled() expect(migrateMocks.activateConfig).toHaveBeenCalled() }) + + it('explains a v3-only database instead of a raw relation-does-not-exist error', async () => { + // Detection missed (e.g. no EQL columns visible) → v2 path — but the + // config table doesn't exist on this database. + migrateMocks.resolveEncryptedColumn.mockResolvedValue(null) + migrateMocks.listEncryptedColumns.mockResolvedValue([]) + queryMock.mockImplementation(async (sql: string) => + typeof sql === 'string' && sql.includes('to_regclass') + ? { rows: [{ exists: null }] } + : { rows: [] }, + ) + const exitSpy = spyExit() + + await cutoverCommand({ table: 'users', column: 'email' }) + + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining('no EQL v2 configuration table'), + ) + expect(migrateMocks.renameEncryptedColumns).not.toHaveBeenCalled() + expect(exitSpy).toHaveBeenCalledWith(1) + exitSpy.mockRestore() + }) }) describe('encrypt drop — EQL version awareness', () => { beforeEach(() => { vi.clearAllMocks() + queryMock.mockResolvedValue({ rows: [] }) + migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_INFO) + migrateMocks.countUnencrypted.mockResolvedValue(0) }) - it('v3: requires phase backfilled and drops the ORIGINAL column', async () => { - migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v3') + it('v3: requires phase backfilled, verifies coverage, and drops the ORIGINAL column', async () => { migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilled' }) await dropCommand({ table: 'users', column: 'email' }) + // The coverage gate ran against the RESOLVED encrypted column. + expect(migrateMocks.countUnencrypted).toHaveBeenCalledWith( + expect.anything(), + 'users', + 'email', + 'email_encrypted', + ) // Non-drizzle fallback writes the SQL file — inspect the generated DDL. const sql = writeFileMock.mock.calls[0]?.[1] as string expect(sql).toContain('DROP COLUMN "email"') @@ -133,12 +246,44 @@ describe('encrypt drop — EQL version awareness', () => { ) }) + it('v3: refuses to generate when rows are still plaintext-only', async () => { + migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilled' }) + migrateMocks.countUnencrypted.mockResolvedValueOnce(7) + const exitSpy = spyExit() + + await dropCommand({ table: 'users', column: 'email' }) + + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining('7 row(s)'), + ) + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining('stash encrypt backfill'), + ) + expect(writeFileMock).not.toHaveBeenCalled() + expect(migrateMocks.appendEvent).not.toHaveBeenCalled() + expect(exitSpy).toHaveBeenCalledWith(1) + exitSpy.mockRestore() + }) + + it('v3: gates coverage on the RESOLVED encrypted column name', async () => { + migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_CUSTOM_INFO) + migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilled' }) + + await dropCommand({ table: 'users', column: 'email' }) + + expect(migrateMocks.countUnencrypted).toHaveBeenCalledWith( + expect.anything(), + 'users', + 'email', + 'email_enc', + ) + const sql = writeFileMock.mock.calls[0]?.[1] as string + expect(sql).toContain('DROP COLUMN "email"') + }) + it('v3: rejects when not yet backfilled', async () => { - migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v3') migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilling' }) - const exitSpy = vi - .spyOn(process, 'exit') - .mockImplementation((() => undefined) as never) + const exitSpy = spyExit() await dropCommand({ table: 'users', column: 'email' }) @@ -150,13 +295,14 @@ describe('encrypt drop — EQL version awareness', () => { exitSpy.mockRestore() }) - it('v2: unchanged — requires cut-over and drops _plaintext (regression pin)', async () => { - migrateMocks.detectColumnEqlVersion.mockResolvedValueOnce('v2') + it('v2: unchanged — requires cut-over, no coverage gate, drops _plaintext (regression pin)', async () => { + migrateMocks.resolveEncryptedColumn.mockResolvedValue(V2_INFO) migrateMocks.progress.mockResolvedValueOnce({ phase: 'cut-over' }) await dropCommand({ table: 'users', column: 'email' }) const sql = writeFileMock.mock.calls[0]?.[1] as string expect(sql).toContain('DROP COLUMN "email_plaintext"') + expect(migrateMocks.countUnencrypted).not.toHaveBeenCalled() }) }) diff --git a/packages/cli/src/commands/encrypt/backfill.ts b/packages/cli/src/commands/encrypt/backfill.ts index 7051ad89e..7f7edb006 100644 --- a/packages/cli/src/commands/encrypt/backfill.ts +++ b/packages/cli/src/commands/encrypt/backfill.ts @@ -1,6 +1,7 @@ import { appendEvent, detectColumnEqlVersion, + type EqlVersion, type ManifestColumn, progress, runBackfill, @@ -149,7 +150,7 @@ export async function backfillCommand(options: BackfillCommandOptions) { ) if (eqlVersion) { p.log.info( - `${options.table}.${encryptedColumn} is EQL ${eqlVersion}${eqlVersion === 'v3' ? ' — lifecycle is backfill → switch the app to the encrypted column by name → drop (no cut-over rename).' : ''}`, + `${options.table}.${encryptedColumn} is EQL v${eqlVersion}${eqlVersion === 3 ? ' — lifecycle is backfill → switch the app to the encrypted column by name → drop (no cut-over rename).' : ''}`, ) } @@ -202,6 +203,7 @@ export async function backfillCommand(options: BackfillCommandOptions) { column, schemaColumnKey, plaintextColumn, + encryptedColumn, options.pkColumn, eqlVersion, ) @@ -272,7 +274,7 @@ export async function backfillCommand(options: BackfillCommandOptions) { return } - if (eqlVersion === 'v3') { + if (eqlVersion === 3) { p.note( `EQL v3 has no cut-over. Next:\n 1. Point your application at ${encryptedColumn} (schema + queries), deploy, verify reads.\n 2. Generate the plaintext drop: stash encrypt drop --table ${options.table} --column ${plaintextColumn}`, 'Next steps (EQL v3)', @@ -522,8 +524,9 @@ function buildManifestEntry( column: ColumnSchema | undefined, schemaColumnKey: string, plaintextColumn: string, + encryptedColumn: string, pkColumn: string | undefined, - eqlVersion: 'v2' | 'v3' | null, + eqlVersion: EqlVersion | null, ): ManifestColumn { // SDK `cast_as` ('string', 'number', …) and EQL `castAs` ('text', // 'double', …) are different vocabularies; translate via the same @@ -541,16 +544,23 @@ function buildManifestEntry( (kind) => indexConfig[kind] !== undefined, ) - return { + const entry: ManifestColumn = { column: plaintextColumn, castAs, indexes, + // Recorded so later commands (cutover/drop/status) don't have to guess + // the name from the `_encrypted` convention — the name is a + // convention only, never relied upon. + encryptedColumn, // v2's ladder ends with the rename cut-over; v3 has none — its end // state is the plaintext column dropped. - targetPhase: eqlVersion === 'v3' ? 'dropped' : 'cut-over', - ...(pkColumn ? { pkColumn } : {}), - ...(eqlVersion ? { eqlVersion: eqlVersion === 'v3' ? 3 : 2 } : {}), - } as ManifestColumn + targetPhase: eqlVersion === 3 ? 'dropped' : 'cut-over', + } + if (pkColumn) entry.pkColumn = pkColumn + // Absent means UNKNOWN (detection couldn't see the column), not v2 — + // readers fall back to the domain type in the database. + if (eqlVersion) entry.eqlVersion = eqlVersion + return entry } // Drop the wrapping default so unknown values fail validation instead of diff --git a/packages/cli/src/commands/encrypt/cutover.ts b/packages/cli/src/commands/encrypt/cutover.ts index 733d6bb71..ca9624dca 100644 --- a/packages/cli/src/commands/encrypt/cutover.ts +++ b/packages/cli/src/commands/encrypt/cutover.ts @@ -1,7 +1,6 @@ import { activateConfig, appendEvent, - detectColumnEqlVersion, migrateConfig, progress, reloadConfig, @@ -13,6 +12,7 @@ import { detectDrizzle } from '@/commands/db/detect.js' import { detectPackageManager, runnerCommand } from '@/commands/init/utils.js' import { loadStashConfig } from '@/config/index.js' import { scaffoldDrizzleMigration } from './drizzle-helper.js' +import { resolveColumnLifecycle } from './lib/resolve-eql.js' /** * Options accepted by `stash encrypt cutover`. Swaps the plaintext and @@ -66,24 +66,36 @@ export async function cutoverCommand(options: CutoverCommandOptions) { // Cut-over is an EQL v2 concept: v2 hides the swap behind // `eql_v2.rename_encrypted_columns()` + a Proxy config promotion. A v3 // column has neither — the application switches to the encrypted column - // BY NAME, and the plaintext column is dropped later. Detect before any - // phase/config checks so v3 users get the real answer, not a confusing - // precondition error. - const encryptedColumn = `${options.column}_encrypted` - const version = await detectColumnEqlVersion( + // BY NAME, and the plaintext column is dropped later. Resolve from the + // DOMAIN TYPES (manifest name as a hint; the `_encrypted` naming is + // a convention, never relied upon) before any phase/config checks so v3 + // users get the real answer, not a confusing precondition error. + const { info } = await resolveColumnLifecycle( client, options.table, - encryptedColumn, + options.column, ) - if (version === 'v3') { + const state = await progress(client, options.table, options.column) + + if (info?.version === 3) { + const encryptedColumn = info.column + if (state?.phase !== 'backfilled') { + // Not a "nothing to do" — the user isn't ready for ANY next step + // yet. Exit 1 so scripted pipelines gating on cutover don't read + // an incomplete backfill as success. + p.log.error( + `Cut-over is not applicable to EQL v3 columns, and ${options.table}.${options.column} hasn't finished backfilling (phase '${state?.phase ?? '—'}'). Finish the backfill first:\n stash encrypt backfill --table ${options.table} --column ${options.column}`, + ) + exitCode = 1 + return + } p.log.info( - `Cut-over is not applicable to EQL v3 columns. ${options.table}.${encryptedColumn} is EQL v3: there is no rename step — point your application at ${encryptedColumn} (update your schema/queries), verify reads, then generate the plaintext drop with:\n stash encrypt drop --table ${options.table} --column ${options.column}`, + `Cut-over is not applicable to EQL v3 columns. ${options.table}.${encryptedColumn} is EQL v3 (${info.domain}): there is no rename step — point your application at ${encryptedColumn} (update your schema/queries), verify reads, then generate the plaintext drop with:\n stash encrypt drop --table ${options.table} --column ${options.column}`, ) p.outro('Nothing to do for EQL v3.') return } - const state = await progress(client, options.table, options.column) if (state?.phase !== 'backfilled') { p.log.error( `Cannot cut over: ${options.table}.${options.column} is in phase '${state?.phase ?? '—'}'. Must be 'backfilled'.`, @@ -92,6 +104,21 @@ export async function cutoverCommand(options: CutoverCommandOptions) { return } + // Guard the v2 config machinery's existence before querying it: on a + // v3-only database (v3 is the default install) there is no + // `eql_v2_configuration` relation, and an unguarded query surfaces as a + // raw "relation does not exist" error instead of an explanation. + const configTable = await client.query<{ exists: string | null }>( + "SELECT to_regclass('public.eql_v2_configuration')::text AS exists", + ) + if (configTable.rows[0]?.exists == null) { + p.log.error( + `This database has no EQL v2 configuration table — it looks like an EQL v3-only install. Cut-over only applies to EQL v2 columns; for v3, point your application at the encrypted column by name, then generate the plaintext drop with:\n stash encrypt drop --table ${options.table} --column ${options.column}`, + ) + exitCode = 1 + return + } + // Verify a pending EQL config exists. cutover assumes the user has // already run `stash db push` against a schema that switches the // column from `_encrypted` (or whatever twin name) to `` — diff --git a/packages/cli/src/commands/encrypt/drop.ts b/packages/cli/src/commands/encrypt/drop.ts index b385235d8..321bd8662 100644 --- a/packages/cli/src/commands/encrypt/drop.ts +++ b/packages/cli/src/commands/encrypt/drop.ts @@ -2,7 +2,7 @@ import fs from 'node:fs' import path from 'node:path' import { appendEvent, - detectColumnEqlVersion, + countUnencrypted, progress, setManifestTargetPhase, } from '@cipherstash/migrate' @@ -12,6 +12,7 @@ import { detectDrizzle } from '@/commands/db/detect.js' import { detectPackageManager, runnerCommand } from '@/commands/init/utils.js' import { loadStashConfig } from '@/config/index.js' import { scaffoldDrizzleMigration } from './drizzle-helper.js' +import { resolveColumnLifecycle } from './lib/resolve-eql.js' /** * Options accepted by `stash encrypt drop`. Generates a migration file @@ -61,15 +62,18 @@ export async function dropCommand(options: DropCommandOptions) { // The plaintext column's name depends on the EQL version's lifecycle: // v2's cut-over renames `` → `_plaintext` (so that's what we - // drop, after `cut-over`); v3 has no rename — the app switched to - // `_encrypted` by name, so the original `` IS the plaintext - // column, droppable straight after `backfilled`. - const version = await detectColumnEqlVersion( + // drop, after `cut-over`); v3 has no rename — the app switched to the + // encrypted column by name, so the original `` IS the plaintext + // column, droppable straight after `backfilled`. The version and the + // encrypted column's name are resolved from the DOMAIN TYPES (manifest + // name as a hint) — the `_encrypted` naming is a convention only. + const { info } = await resolveColumnLifecycle( client, options.table, - `${options.column}_encrypted`, + options.column, ) - const isV3 = version === 'v3' + const isV3 = info?.version === 3 + const encryptedColumn = info?.column ?? `${options.column}_encrypted` const requiredPhase = isV3 ? 'backfilled' : 'cut-over' const plaintextToDrop = isV3 ? options.column @@ -85,8 +89,29 @@ export async function dropCommand(options: DropCommandOptions) { } if (isV3) { + // The phase gate above proves a backfill FINISHED at some point; it + // says nothing about rows written since (a bulk import or a service + // that isn't dual-writing leaves plaintext-only rows). Dropping the + // original column is the one irreversible step in the v3 ladder, so + // verify live coverage before generating the migration. + const unencrypted = await countUnencrypted( + client, + options.table, + options.column, + encryptedColumn, + ) + if (unencrypted > 0) { + p.log.error( + `Refusing to generate the drop migration: ${unencrypted} row(s) in ${options.table} have "${options.column}" set but "${encryptedColumn}" NULL — dropping "${options.column}" would permanently destroy that data. Likely rows written without dual-writes since the backfill. Re-run:\n stash encrypt backfill --table ${options.table} --column ${options.column}\nthen generate the drop again.`, + ) + exitCode = 1 + return + } + p.log.success( + `Verified: no rows with "${options.column}" set and "${encryptedColumn}" NULL.`, + ) p.log.info( - `${options.table}.${options.column}_encrypted is EQL v3 — the drop targets the original plaintext column "${options.column}" (v3 has no rename, so there is no "${options.column}_plaintext"). Make sure your application reads/writes ${options.column}_encrypted before applying this migration.`, + `${options.table}.${encryptedColumn} is EQL v3 (${info?.domain}) — the drop targets the original plaintext column "${options.column}" (v3 has no rename, so there is no "${options.column}_plaintext"). Make sure your application reads/writes ${encryptedColumn} before applying this migration.`, ) } @@ -95,7 +120,7 @@ export async function dropCommand(options: DropCommandOptions) { dot >= 0 ? `"${options.table.slice(0, dot)}"."${options.table.slice(dot + 1)}"` : `"${options.table}"` - const dropSql = `-- Generated by stash encrypt drop\n-- Drops the plaintext column now that ${options.table}.${options.column}${isV3 ? '_encrypted' : ''} is encrypted.\n\nALTER TABLE ${qualifiedTable} DROP COLUMN "${plaintextToDrop}";\n` + const dropSql = `-- Generated by stash encrypt drop\n-- Drops the plaintext column now that ${options.table}.${isV3 ? encryptedColumn : options.column} is encrypted.\n\nALTER TABLE ${qualifiedTable} DROP COLUMN "${plaintextToDrop}";\n` const cwd = process.cwd() const migrationsDir = options.migrationsDir ?? 'drizzle' diff --git a/packages/cli/src/commands/encrypt/lib/db-readers.ts b/packages/cli/src/commands/encrypt/lib/db-readers.ts index cd64365db..f0e214ec1 100644 --- a/packages/cli/src/commands/encrypt/lib/db-readers.ts +++ b/packages/cli/src/commands/encrypt/lib/db-readers.ts @@ -82,7 +82,12 @@ export async function fetchActiveEqlConfig( } /** - * Read `information_schema.columns` and group column names by table. + * Read `information_schema.columns` and group columns by table, mapping each + * column name to its DOMAIN type (or `null` for plain types). The domain is + * what makes EQL columns self-describing (`eql_v2_encrypted` / `eql_v3_*`), + * so callers can classify encryption state from the types themselves rather + * than relying on the `_encrypted` naming convention. + * * When `tables` is provided the query is constrained to that set — * status's quest log only ever needs ~5 specific tables, so passing * the manifest's tables avoids a full-schema scan. @@ -90,25 +95,32 @@ export async function fetchActiveEqlConfig( export async function fetchPhysicalColumns( client: pg.ClientBase, tables?: ReadonlyArray, -): Promise>> { - const out = new Map>() +): Promise>> { + const out = new Map>() + type Row = { + table_name: string + column_name: string + domain_name: string | null + } try { const result = tables === undefined - ? await client.query<{ table_name: string; column_name: string }>( - `SELECT table_name, column_name FROM information_schema.columns + ? await client.query( + `SELECT table_name, column_name, domain_name + FROM information_schema.columns WHERE table_schema = current_schema()`, ) - : await client.query<{ table_name: string; column_name: string }>( - `SELECT table_name, column_name FROM information_schema.columns + : await client.query( + `SELECT table_name, column_name, domain_name + FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ANY($1::text[])`, [tables], ) for (const row of result.rows) { - const set = out.get(row.table_name) ?? new Set() - set.add(row.column_name) - out.set(row.table_name, set) + const cols = out.get(row.table_name) ?? new Map() + cols.set(row.column_name, row.domain_name) + out.set(row.table_name, cols) } } catch { // information_schema is always present; failures here are surprising diff --git a/packages/cli/src/commands/encrypt/lib/resolve-eql.ts b/packages/cli/src/commands/encrypt/lib/resolve-eql.ts new file mode 100644 index 000000000..505df38a9 --- /dev/null +++ b/packages/cli/src/commands/encrypt/lib/resolve-eql.ts @@ -0,0 +1,58 @@ +import { + type EncryptedColumnInfo, + listEncryptedColumns, + readManifest, + resolveEncryptedColumn, +} from '@cipherstash/migrate' +import type pg from 'pg' + +/** + * The resolved encryption lifecycle for one plaintext column: which column + * carries the ciphertext, and which EQL generation its domain type declares. + */ +export interface ResolvedLifecycle { + /** The encrypted counterpart, or `null` when none could be identified. */ + info: EncryptedColumnInfo | null + /** + * Every EQL-domain column on the table — non-empty when resolution failed + * because several candidates exist and none is identifiable. Lets the + * caller name them instead of erroring blind. + */ + candidates: EncryptedColumnInfo[] +} + +/** + * Resolve a plaintext column's encrypted counterpart, trusting the DOMAIN + * TYPES in the database — the EQL v3 types are self-describing, so the + * `_encrypted` naming is a convention only, never enforced and never + * relied upon. Resolution order: + * + * 1. The manifest's recorded `encryptedColumn` (written by `encrypt + * backfill`, including any `--encrypted-column` override) — used as a + * HINT and still validated against the actual domain type. + * 2. The `_encrypted` convention, validated the same way. + * 3. The table's sole EQL-domain column, if there is exactly one. + * + * The VERSION always comes from the domain type — the manifest's cached + * `eqlVersion` is for display paths that have no DB connection. + */ +export async function resolveColumnLifecycle( + client: pg.ClientBase, + table: string, + column: string, +): Promise { + const manifest = await readManifest().catch(() => null) + const hint = manifest?.tables[table]?.find( + (entry) => entry.column === column, + )?.encryptedColumn + + let info = hint + ? await resolveEncryptedColumn(client, table, column, hint) + : null + // A stale hint (column since renamed/retyped) must not mask a resolvable + // counterpart — fall back to convention + sole-EQL-column resolution. + if (!info) info = await resolveEncryptedColumn(client, table, column) + if (info) return { info, candidates: [] } + + return { info: null, candidates: await listEncryptedColumns(client, table) } +} diff --git a/packages/cli/src/commands/encrypt/status.ts b/packages/cli/src/commands/encrypt/status.ts index 5aeeb0a34..0554f74dc 100644 --- a/packages/cli/src/commands/encrypt/status.ts +++ b/packages/cli/src/commands/encrypt/status.ts @@ -1,4 +1,8 @@ -import { type MigrationPhase, readManifest } from '@cipherstash/migrate' +import { + classifyEqlDomain, + type MigrationPhase, + readManifest, +} from '@cipherstash/migrate' import * as p from '@clack/prompts' import pg from 'pg' import { detectPackageManager, runnerCommand } from '@/commands/init/utils.js' @@ -64,8 +68,9 @@ export async function statusCommand() { intentIndexes: column.indexes, state: stateMap.get(key) ?? null, eqlColumn: eqlConfig.get(key) ?? null, - physicalColumns: physicalCols.get(tableName) ?? new Set(), + physicalColumns: physicalCols.get(tableName) ?? new Map(), eqlVersion: column.eqlVersion, + encryptedColumn: column.encryptedColumn, }), ) } @@ -87,7 +92,7 @@ export async function statusCommand() { intentIndexes: undefined, state, eqlColumn: eqlConfig.get(key) ?? null, - physicalColumns: physicalCols.get(tableName) ?? new Set(), + physicalColumns: physicalCols.get(tableName) ?? new Map(), }), ) } @@ -117,17 +122,22 @@ function renderRow(input: { tableName: string columnName: string intentIndexes: string[] | undefined - /** From the manifest (recorded at backfill time). v3 columns have no - * `eql_v2_configuration` row and no rename, so the v2 drift flags don't - * apply. Absent = written by pre-v3 tooling = v2. */ + /** From the manifest (recorded at backfill time) — a cached HINT. The + * domain type observed in `information_schema` is the source of truth + * and wins below: v3 columns have no `eql_v2_configuration` row and no + * rename, so the v2 drift flags don't apply. */ eqlVersion?: 2 | 3 + /** The encrypted counterpart's name from the manifest, when recorded. + * `_encrypted` is only the conventional fallback. */ + encryptedColumn?: string state: { phase: MigrationPhase rowsProcessed: number | null rowsTotal: number | null } | null eqlColumn: EqlColumnInfo | null - physicalColumns: Set + /** Column name → domain type (or null) from `information_schema`. */ + physicalColumns: ReadonlyMap }): Row { const { tableName, @@ -139,9 +149,21 @@ function renderRow(input: { } = input const phase = state?.phase ?? (intentIndexes ? 'schema-added' : '—') + + // Version resolution: the encrypted column's DOMAIN TYPE is + // self-describing and wins; the manifest's cached eqlVersion covers the + // window where the physical column isn't visible. The manifest's recorded + // encryptedColumn is preferred over the naming convention. + const encryptedName = input.encryptedColumn ?? `${columnName}_encrypted` + const physicalDomain = physicalColumns.get(encryptedName) + const physicalVersion = physicalDomain + ? classifyEqlDomain(physicalDomain) + : null + const version = physicalVersion ?? input.eqlVersion ?? null + // v3 columns have no eql_v2_configuration row by design — show the // version rather than a misleading blank. - const eql = eqlColumn ? eqlColumn.state : input.eqlVersion === 3 ? 'v3' : '—' + const eql = eqlColumn ? eqlColumn.state : version === 3 ? 'v3' : '—' const indexes = eqlColumn ? eqlColumn.indexes.join(', ') || '(none)' : intentIndexes?.join(', ') || '—' @@ -154,13 +176,13 @@ function renderRow(input: { // writes covered every row from seeding). Frame per phase. const progress = formatProgress(phase, state) - const isV3 = input.eqlVersion === 3 + const isV3 = version === 3 const flags: string[] = [] // v2-only drift flags: a v3 column is never registered in // `eql_v2_configuration` (no config table exists) and never reaches the // rename that creates `_plaintext`. if (!isV3 && intentIndexes && !eqlColumn) flags.push('not-registered') - if (intentIndexes && !physicalColumns.has(`${columnName}_encrypted`)) { + if (intentIndexes && !physicalColumns.has(encryptedName)) { flags.push('encrypted-col-missing') } if ( diff --git a/packages/cli/src/commands/init/lib/__tests__/setup-prompt.test.ts b/packages/cli/src/commands/init/lib/__tests__/setup-prompt.test.ts index 815c7f71a..9211804e2 100644 --- a/packages/cli/src/commands/init/lib/__tests__/setup-prompt.test.ts +++ b/packages/cli/src/commands/init/lib/__tests__/setup-prompt.test.ts @@ -38,12 +38,16 @@ describe('renderSetupPrompt — orient + route (implement mode)', () => { expect(out).toContain('### Converting in place is not supported') }) - it('frames the migrate-existing flow as encryption rollout + cutover with a deploy gate', () => { + it('frames the migrate-existing flow as rollout + backfill-and-switch with a deploy gate', () => { // The whole point of the rewrite. No "phase" jargon; explicit deploy - // gate banner; rollout vs cutover named sections. + // gate banner; named sections. The switch step is EQL-version-aware: + // v3 (the default) has no rename — the app points at the encrypted + // column by name; cutover is the v2 rename path. const out = renderSetupPrompt(baseCtx) expect(out).toMatch(/encryption rollout/i) - expect(out).toMatch(/encryption cutover/i) + expect(out).toMatch(/backfill and switch/i) + expect(out).toMatch(/EQL v3 \(the default\)/) + expect(out).toMatch(/encrypt cutover/) expect(out).toMatch(/deploy gate/i) expect(out).not.toMatch(/phase 1|phase 2|four-deploy/i) }) diff --git a/packages/cli/src/commands/init/lib/setup-prompt.ts b/packages/cli/src/commands/init/lib/setup-prompt.ts index b278bdd0e..550e0cff3 100644 --- a/packages/cli/src/commands/init/lib/setup-prompt.ts +++ b/packages/cli/src/commands/init/lib/setup-prompt.ts @@ -254,7 +254,7 @@ export function renderImplementPrompt(ctx: SetupPromptContext): string { '', "Use when the column **already exists** in the user's database and contains live data that must be preserved.", '', - "Why it's staged: there is no atomic way to replace a populated column with an encrypted one without corrupting data. Instead, taking encryption to production happens in two passes around a deploy gate. The first pass — the **encryption rollout** — adds the encrypted twin column and the dual-write code; the user deploys that to production so every new write produces both plaintext and ciphertext. The second pass — the **encryption cutover** — backfills historical rows, renames the encrypted twin into the original column name, switches reads through the encryption client, and drops the old plaintext column.", + "Why it's staged: there is no atomic way to replace a populated column with an encrypted one without corrupting data. Instead, taking encryption to production happens in two passes around a deploy gate. The first pass — the **encryption rollout** — adds the encrypted twin column and the dual-write code; the user deploys that to production so every new write produces both plaintext and ciphertext. The second pass backfills historical rows and switches reads to the encrypted column: on EQL v3 (the default) the application points at the encrypted column **by name** and the old plaintext column is dropped — there is no rename step; on EQL v2 a **cutover** renames the encrypted twin into the original column name first.", '', '#### Encryption rollout — what lands before the deploy', '', @@ -267,13 +267,13 @@ export function renderImplementPrompt(ctx: SetupPromptContext): string { '', `to confirm where they are, then \`${cli} plan\` to draft the cutover. Do not run \`${cli} encrypt backfill\`, \`${cli} encrypt cutover\`, or \`${cli} encrypt drop\` until that has happened — \`${cli} impl\` will refuse to run cutover-step plans without a recorded \`dual_writing\` event.`, '', - '#### Encryption cutover — after dual-writes are live', + '#### Backfill and switch — after dual-writes are live', '', `3. **Backfill.** Run \`${cli} encrypt backfill --table --column \`. The CLI prompts the user (or accepts \`--confirm-dual-writes-deployed\` non-interactively) to confirm dual-writes are live, then chunks through the existing rows. Resumable; checkpoints to \`cs_migrations\` after every chunk. SIGINT-safe.`, - `4. **Switch the schema, then cutover.** Update the schema file to declare the encrypted column under its final name (drop \`_encrypted\` suffix, switch \`\` to \`encryptedType\`). Then \`${cli} encrypt cutover --table --column \` runs the rename in one transaction (\`\` → \`_plaintext\`, \`_encrypted\` → \`\`).`, - '5. **Wire the read path through the encryption client.** Post-cutover, `` holds ciphertext. Read code paths must decrypt before returning the value to callers — `decryptModel(row, table)` for Drizzle, the `encryptedSupabase` wrapper for Supabase, or the equivalent `decrypt`/`bulkDecryptModels` calls. Without this step, your read paths return raw encrypted payloads to end users. The integration skill has the exact API.', - '6. **Remove the dual-write code.** The plaintext column is now `_plaintext` and is no longer authoritative. Delete the dual-write logic from the persistence layer.', - `7. **Drop.** Run \`${cli} encrypt drop --table --column \`. Generates a migration that removes the now-unused \`_plaintext\`. Apply with the project's normal migration tooling.`, + `4. **Switch reads to the encrypted column.** The step depends on the EQL version (\`${cli} encrypt backfill\` prints it; \`${cli} encrypt status\` shows it). **EQL v3 (the default):** there is no rename — update the schema and queries to read/write the encrypted column by its own name, and wire decryption through the encryption client. **EQL v2:** update the schema file to declare the encrypted column under its final name (drop the twin suffix, switch \`\` to \`encryptedType\`), then \`${cli} encrypt cutover --table --column \` runs the rename in one transaction (\`\` → \`_plaintext\`, twin → \`\`).`, + '5. **Wire the read path through the encryption client.** The read column now holds ciphertext. Read code paths must decrypt before returning the value to callers — `decryptModel(row, table)` for Drizzle, the `encryptedSupabase` wrapper for Supabase, or the equivalent `decrypt`/`bulkDecryptModels` calls. Without this step, your read paths return raw encrypted payloads to end users. The integration skill has the exact API.', + '6. **Remove the dual-write code.** The plaintext column (still `` on v3; renamed `_plaintext` on v2) is no longer authoritative. Delete the dual-write logic from the persistence layer.', + `7. **Drop.** Run \`${cli} encrypt drop --table --column \`. Generates a migration that removes the now-unused plaintext column (on v3 it first verifies no rows are still plaintext-only). Apply with the project's normal migration tooling.`, '', 'Recovery: if the user reports that backfill ran *before* the dual-write code was actually live, drift is expected (rows written during the backfill window land in plaintext only). Re-run with `--force` to encrypt every plaintext row regardless of current state.', '', diff --git a/packages/cli/src/commands/status/index.ts b/packages/cli/src/commands/status/index.ts index 89b8b8e35..08063a426 100644 --- a/packages/cli/src/commands/status/index.ts +++ b/packages/cli/src/commands/status/index.ts @@ -33,11 +33,17 @@ const CONNECT_TIMEOUT_MS = 2_000 function manifestColumns( manifest: Manifest, -): { table: string; column: string }[] { - const out: { table: string; column: string }[] = [] +): { table: string; column: string; eqlVersion?: 2 | 3 }[] { + const out: { table: string; column: string; eqlVersion?: 2 | 3 }[] = [] for (const [table, cols] of Object.entries(manifest.tables)) { for (const col of cols) { - out.push({ table, column: col.column }) + out.push({ + table, + column: col.column, + // Cached hint only — the quest ladder shape (4 rungs for v3, 5 for + // v2) is display, so the manifest's record is good enough here. + ...(col.eqlVersion ? { eqlVersion: col.eqlVersion } : {}), + }) } } return out @@ -115,10 +121,11 @@ export async function gatherObservations( return { table: c.table, column: c.column, + ...(c.eqlVersion ? { eqlVersion: c.eqlVersion } : {}), phase: phaseRow ? phaseRow.phase : null, eql: eqlInfo ? { state: eqlInfo.state } : null, physicalEncryptedTwinExists: ( - physicalCols.get(c.table) ?? new Set() + physicalCols.get(c.table) ?? new Map() ).has(`${c.column}_encrypted`), } }) diff --git a/packages/cli/src/commands/status/quest.ts b/packages/cli/src/commands/status/quest.ts index 0dabf8d34..b4225e4ef 100644 --- a/packages/cli/src/commands/status/quest.ts +++ b/packages/cli/src/commands/status/quest.ts @@ -75,6 +75,11 @@ export interface ColumnObservation { * cs_migrations doesn't yet track this column. `undefined` when the * caller can't tell. */ physicalEncryptedTwinExists?: boolean + /** The column's EQL generation, from the manifest's cached `eqlVersion`. + * v3 has a 4-objective ladder (no cut-over rename — the app switches to + * the encrypted column by name). `undefined` (unknown / pre-v3 manifest) + * renders the v2 ladder. */ + eqlVersion?: 2 | 3 } const MIGRATE_OBJECTIVES = [ @@ -85,6 +90,16 @@ const MIGRATE_OBJECTIVES = [ 'Drop plaintext column', ] +// EQL v3 has no cut-over: configuration lives in the column's own domain +// type, so the app switches to the encrypted column BY NAME and the +// plaintext column is dropped straight after backfill. +const MIGRATE_OBJECTIVES_V3 = [ + 'Schema-add — encrypted column added', + 'Dual-writes deployed to production', + 'Backfill historical rows', + 'Switch app to the encrypted column, then drop plaintext', +] + const NEW_OBJECTIVES = [ 'Schema-add — encrypted column declared and migrated', 'Live in active EQL config', @@ -119,7 +134,12 @@ export function buildColumnQuest( cli: string, ): ColumnQuest { const path = inferQuestPath(obs) - const labels = path === 'migrate' ? MIGRATE_OBJECTIVES : NEW_OBJECTIVES + const labels = + path === 'migrate' + ? obs.eqlVersion === 3 + ? MIGRATE_OBJECTIVES_V3 + : MIGRATE_OBJECTIVES + : NEW_OBJECTIVES const total = labels.length const doneCount = computeDoneCount(path, obs) const dbUnreachable = obs.phase === undefined && obs.eql === undefined @@ -188,10 +208,13 @@ function computeDoneNew(obs: ColumnObservation): number { function computeDoneMigrate(obs: ColumnObservation): number { if (obs.phase === undefined && obs.eql === undefined) return 0 - // Phase progression dominates when we have it. + const isV3 = obs.eqlVersion === 3 + + // Phase progression dominates when we have it. The v3 ladder is one rung + // shorter (no cut-over), so its terminal phases map one lower. switch (obs.phase) { case 'dropped': - return 5 + return isV3 ? 4 : 5 case 'cut-over': return 4 case 'backfilled': @@ -226,7 +249,23 @@ function nextMoveFor( return `Promote the pending EQL config — \`${cli} db activate\`.` } - // Migrate. + // Migrate. The v3 ladder has no cut-over — after backfill the app + // switches to the encrypted column by name, then drops the plaintext. + if (obs.eqlVersion === 3) { + switch (doneCount) { + case 0: + return 'Add the encrypted column and run the migration.' + case 1: + return `Wire dual-write code on every persistence path, deploy to production, then run \`${cli} encrypt backfill\` (it confirms dual-writes and records the event).` + case 2: + return `Run \`${cli} encrypt backfill --table ${obs.table} --column ${obs.column}\` to encrypt historical rows.` + case 3: + return `Point your application at the encrypted column (update schema/queries — EQL v3 has no rename step), verify reads, then run \`${cli} encrypt drop --table ${obs.table} --column ${obs.column}\`.` + default: + return '' + } + } + switch (doneCount) { case 0: return 'Add the encrypted twin column (`_encrypted`) and run the migration.' diff --git a/packages/migrate/README.md b/packages/migrate/README.md index 8456603b5..8a05e22ea 100644 --- a/packages/migrate/README.md +++ b/packages/migrate/README.md @@ -16,7 +16,7 @@ EQL v3: schema-added → dual-writing → backfilling → backfilled ——— State is tracked in an append-only `cipherstash.cs_migrations` table installed by `stash eql install`. - **EQL v2** additionally keeps its intent (indexes, cast_as) in `eql_v2_configuration` so CipherStash Proxy works against the same database, and finishes with a **cut-over**: `eql_v2.rename_encrypted_columns()` swaps `_encrypted` into place (`` becomes `_plaintext`) alongside a config promotion. -- **EQL v3** has **no configuration table and no cut-over** — each column's domain type encodes its own configuration. The application switches to `_encrypted` *by name*, and the original plaintext `` is dropped once verified. Backfill verification is a plain count of the populated target column (`countEncrypted`); the concrete `eql_v3_*` domain's CHECK constraint guarantees every non-null value is a valid v3 envelope. +- **EQL v3** has **no configuration table and no cut-over** — each column's domain type encodes its own configuration. The v3 types are *self-describing*, so tooling resolves encrypted columns from the domain types themselves; the `_encrypted` naming is a convention only, never enforced or relied upon (`resolveEncryptedColumn`). The application switches to the encrypted column *by name*, and the original plaintext `` is dropped once verified: `stash encrypt drop` refuses to generate the migration while any row still has the plaintext set and the encrypted column NULL (`countUnencrypted`); the concrete `eql_v3_*` domain's CHECK constraint guarantees every non-null value is a valid v3 envelope. ## API @@ -57,9 +57,13 @@ Direct access to the `cs_migrations` event log. Use these if you're building you Thin wrappers around `eql_v2.rename_encrypted_columns()` (the **v2** cut-over primitive) and `eql_v2.reload_config()` (Proxy refresh hint — no-op when connected directly to Postgres). Not used in the v3 lifecycle — v3 has no rename step. -### `detectColumnEqlVersion(client, table, column)` / `countEncrypted(client, table, column)` +### `detectColumnEqlVersion` / `resolveEncryptedColumn` / `listEncryptedColumns` / `classifyEqlDomain` -`detectColumnEqlVersion` inspects the column's Postgres domain type and returns `'v2'`, `'v3'`, or `null` (not an EQL column) — the branch point for everything version-specific above. `countEncrypted` counts populated target-column rows: the v3 backfill-verification primitive (v2 verified through `eql_v2.count_encrypted_with_active_config`, which needs the config table v3 doesn't have). +The EQL types are self-describing, and these are the domain-type primitives everything version-specific above branches on. `detectColumnEqlVersion(client, table, column)` inspects one column's Postgres domain type and returns `2`, `3`, or `null` (not an EQL column); resolution is case-exact (quoted-identifier semantics, matching the rest of the pipeline) and honours `search_path`. `resolveEncryptedColumn(client, table, plaintextColumn, hint?)` finds a plaintext column's encrypted counterpart from the domain types — an explicit hint (e.g. the manifest's recorded `encryptedColumn`) wins, then the `_encrypted` convention, then the table's sole EQL column; the name is never assumed. `listEncryptedColumns` returns every EQL-domain column on a table, classified. + +### `countEncrypted` / `countUnencrypted` + +Coverage counts over the live table. `countUnencrypted(client, table, plaintextColumn, encryptedColumn)` counts rows with plaintext set and ciphertext NULL — the check `stash encrypt drop` runs before generating the v3 plaintext-drop migration (a non-zero count means rows were written without dual-writes since the backfill). `countEncrypted` counts populated target-column rows (v2 verifies through `eql_v2.count_encrypted_with_active_config`, which needs the config table v3 doesn't have). Both are full-table scans — fine as one-shot verification, not per-row primitives. ### `readManifest(cwd)` / `writeManifest(manifest, cwd)` diff --git a/packages/migrate/src/__tests__/backfill-v3.integration.test.ts b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts index 91e5c58ec..aa61071bb 100644 --- a/packages/migrate/src/__tests__/backfill-v3.integration.test.ts +++ b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts @@ -31,6 +31,11 @@ import { type EncryptionClientLike, runBackfill } from '../backfill.js' import { countEncrypted } from '../cursor.js' import { installMigrationsSchema } from '../install.js' import { progress } from '../state.js' +import { + detectColumnEqlVersion, + listEncryptedColumns, + resolveEncryptedColumn, +} from '../version.js' const PG_URL = process.env.PG_TEST_URL const runIntegration = Boolean(PG_URL) @@ -190,3 +195,87 @@ describe.skipIf(!runIntegration)('runBackfill with EQL v3 payloads', () => { } }) }) + +/** + * The domain-type resolution primitives against a REAL Postgres catalog — + * the part unit mocks can't prove: case-exact `to_regclass` resolution for + * quoted (Prisma-style) table names, and name-free discovery of encrypted + * columns from their domain types. Local domains named like EQL's are + * enough: classification keys on `pg_type.typname`, which is what a real + * EQL install produces too. + */ +describe.skipIf(!runIntegration)( + 'EQL version resolution (real catalog)', + () => { + let pool: pg.Pool + + beforeAll(async () => { + pool = new pg.Pool({ connectionString: PG_URL, max: 2 }) + await pool.query('DROP SCHEMA IF EXISTS migrate_v3_resolve CASCADE') + await pool.query('CREATE SCHEMA migrate_v3_resolve') + await pool.query( + 'CREATE DOMAIN migrate_v3_resolve.eql_v3_text_search_t AS jsonb', + ) + // Mixed-case table (Prisma default naming) with a custom-named + // encrypted column: exercises BOTH conventions this module must not + // rely on — lowercase names and the `_encrypted` suffix. + await pool.query(` + CREATE TABLE migrate_v3_resolve."Users" ( + id bigint PRIMARY KEY, + email text, + secret_blob migrate_v3_resolve.eql_v3_text_search_t + ) + `) + }) + + afterAll(async () => { + await pool.query('DROP SCHEMA IF EXISTS migrate_v3_resolve CASCADE') + await pool.end() + }) + + it('detects the version on a mixed-case (quoted) table name', async () => { + expect( + await detectColumnEqlVersion( + pool as unknown as pg.ClientBase, + 'migrate_v3_resolve.Users', + 'secret_blob', + ), + ).toBe(3) + }) + + it('returns null (not an error) for a table that truly does not exist', async () => { + expect( + await detectColumnEqlVersion( + pool as unknown as pg.ClientBase, + 'migrate_v3_resolve.users', + 'secret_blob', + ), + ).toBeNull() + }) + + it('lists encrypted columns from domain types alone', async () => { + expect( + await listEncryptedColumns( + pool as unknown as pg.ClientBase, + 'migrate_v3_resolve.Users', + ), + ).toEqual([ + { + column: 'secret_blob', + domain: 'eql_v3_text_search_t', + version: 3, + }, + ]) + }) + + it('resolves the encrypted counterpart with no naming convention at all', async () => { + const info = await resolveEncryptedColumn( + pool as unknown as pg.ClientBase, + 'migrate_v3_resolve.Users', + 'email', + ) + expect(info?.column).toBe('secret_blob') + expect(info?.version).toBe(3) + }) + }, +) diff --git a/packages/migrate/src/__tests__/version.test.ts b/packages/migrate/src/__tests__/version.test.ts index e95036cec..d0d9a7648 100644 --- a/packages/migrate/src/__tests__/version.test.ts +++ b/packages/migrate/src/__tests__/version.test.ts @@ -1,72 +1,161 @@ -import type { ClientBase, QueryConfig, QueryResult, QueryResultRow } from 'pg' -import { describe, expect, it } from 'vitest' -import { detectColumnEqlVersion } from '../version.js' +import type { ClientBase } from 'pg' +import { describe, expect, it, vi } from 'vitest' +import { + classifyEqlDomain, + detectColumnEqlVersion, + listEncryptedColumns, + resolveEncryptedColumn, +} from '../version.js' -interface RecordedQuery { - text: string - values: unknown[] +function mockClient(rows: Array>) { + const query = vi.fn().mockResolvedValue({ rows }) + return { client: { query } as unknown as ClientBase, query } } -function createMockClient(rows: Array>): { - client: ClientBase - queries: RecordedQuery[] -} { - const queries: RecordedQuery[] = [] - const client = { - query(config: string | QueryConfig, values?: unknown[]) { - const text = typeof config === 'string' ? config : config.text - queries.push({ text, values: values ?? [] }) - return Promise.resolve({ - rows, - rowCount: rows.length, - command: '', - oid: 0, - fields: [], - } as unknown as QueryResult) - }, - } as unknown as ClientBase - return { client, queries } -} - -describe('detectColumnEqlVersion', () => { - it('maps the eql_v2_encrypted domain to v2', async () => { - const { client } = createMockClient([{ domain_name: 'eql_v2_encrypted' }]) - expect( - await detectColumnEqlVersion(client, 'users', 'email_encrypted'), - ).toBe('v2') +describe('classifyEqlDomain', () => { + it('maps eql_v2_encrypted to 2', () => { + expect(classifyEqlDomain('eql_v2_encrypted')).toBe(2) }) - it('maps any concrete eql_v3_* domain to v3', async () => { + it('maps any eql_v3_* domain to 3', () => { for (const domain of [ 'eql_v3_text_search', 'eql_v3_text_match', 'eql_v3_int8_ord', 'eql_v3_encrypted', ]) { - const { client } = createMockClient([{ domain_name: domain }]) - expect( - await detectColumnEqlVersion(client, 'users', 'email_encrypted'), - ).toBe('v3') + expect(classifyEqlDomain(domain)).toBe(3) } }) + it('maps non-EQL types to null', () => { + expect(classifyEqlDomain('text')).toBeNull() + expect(classifyEqlDomain('jsonb')).toBeNull() + expect(classifyEqlDomain('citext')).toBeNull() + }) +}) + +describe('detectColumnEqlVersion', () => { + it('classifies from the domain type', async () => { + const { client } = mockClient([{ domain_name: 'eql_v2_encrypted' }]) + expect( + await detectColumnEqlVersion(client, 'users', 'email_encrypted'), + ).toBe(2) + }) + it('returns null for a plaintext column (base type, not a domain)', async () => { - const { client } = createMockClient([{ domain_name: 'text' }]) + const { client } = mockClient([{ domain_name: 'text' }]) expect(await detectColumnEqlVersion(client, 'users', 'email')).toBeNull() }) it('returns null when the column/table is not found', async () => { - const { client } = createMockClient([]) + const { client } = mockClient([]) expect(await detectColumnEqlVersion(client, 'nope', 'missing')).toBeNull() }) - it('passes the qualified table name and column as bind params (to_regclass)', async () => { - const { client, queries } = createMockClient([ + it('resolves the table case-exactly: quoted-identifier semantics, not raw to_regclass parsing', async () => { + // A bare to_regclass($1) case-folds 'User' to 'user', silently missing + // Prisma-style quoted tables while the rest of the pipeline (which + // quotes identifiers verbatim) works — wedging a v3 column into the v2 + // lifecycle. The format('%I', …) wrapping is what prevents that. + const { client, query } = mockClient([ + { domain_name: 'eql_v3_text_search' }, + ]) + await detectColumnEqlVersion(client, 'User', 'email_encrypted') + const [sql, values] = query.mock.calls[0] as [string, unknown[]] + expect(sql).toContain("format('%I'") + expect(sql).not.toMatch(/to_regclass\(\$1\)/) + expect(values).toEqual(['User', null, 'email_encrypted']) + }) + + it('splits schema-qualified names on the first dot, like qualifyTable', async () => { + const { client, query } = mockClient([ { domain_name: 'eql_v3_text_search' }, ]) - await detectColumnEqlVersion(client, 'app.users', 'email_encrypted') - expect(queries).toHaveLength(1) - expect(queries[0].text).toContain('to_regclass($1)') - expect(queries[0].values).toEqual(['app.users', 'email_encrypted']) + await detectColumnEqlVersion(client, 'app.Users', 'email_encrypted') + const [, values] = query.mock.calls[0] as [string, unknown[]] + expect(values).toEqual(['Users', 'app', 'email_encrypted']) + }) +}) + +describe('listEncryptedColumns', () => { + it('returns only EQL-domain columns, classified', async () => { + const { client } = mockClient([ + { column: 'id', domain_name: 'int8' }, + { column: 'email', domain_name: 'text' }, + { column: 'email_enc', domain_name: 'eql_v3_text_search' }, + { column: 'ssn_encrypted', domain_name: 'eql_v2_encrypted' }, + ]) + expect(await listEncryptedColumns(client, 'users')).toEqual([ + { column: 'email_enc', domain: 'eql_v3_text_search', version: 3 }, + { column: 'ssn_encrypted', domain: 'eql_v2_encrypted', version: 2 }, + ]) + }) +}) + +describe('resolveEncryptedColumn', () => { + const TABLE = [ + { column: 'id', domain_name: 'int8' }, + { column: 'email', domain_name: 'text' }, + ] + + it('an explicit hint wins, validated against the domain type', async () => { + const { client } = mockClient([ + ...TABLE, + { column: 'email_enc', domain_name: 'eql_v3_text_search' }, + { column: 'email_encrypted', domain_name: 'eql_v3_text_eq' }, + ]) + expect( + await resolveEncryptedColumn(client, 'users', 'email', 'email_enc'), + ).toEqual({ column: 'email_enc', domain: 'eql_v3_text_search', version: 3 }) + }) + + it('a hint naming a non-EQL column resolves to null, not a guess', async () => { + const { client } = mockClient([ + ...TABLE, + { column: 'email_encrypted', domain_name: 'eql_v3_text_eq' }, + ]) + expect( + await resolveEncryptedColumn(client, 'users', 'email', 'email'), + ).toBeNull() + }) + + it('falls back to the _encrypted convention', async () => { + const { client } = mockClient([ + ...TABLE, + { column: 'email_encrypted', domain_name: 'eql_v3_text_eq' }, + { column: 'other_encrypted', domain_name: 'eql_v3_text_eq' }, + ]) + expect(await resolveEncryptedColumn(client, 'users', 'email')).toEqual({ + column: 'email_encrypted', + domain: 'eql_v3_text_eq', + version: 3, + }) + }) + + it('resolves a sole EQL column regardless of its name — the convention is never required', async () => { + const { client } = mockClient([ + ...TABLE, + { column: 'secret_blob', domain_name: 'eql_v3_text_search' }, + ]) + expect(await resolveEncryptedColumn(client, 'users', 'email')).toEqual({ + column: 'secret_blob', + domain: 'eql_v3_text_search', + version: 3, + }) + }) + + it('returns null when several EQL columns exist and none is identifiable', async () => { + const { client } = mockClient([ + ...TABLE, + { column: 'a_enc', domain_name: 'eql_v3_text_eq' }, + { column: 'b_enc', domain_name: 'eql_v3_text_eq' }, + ]) + expect(await resolveEncryptedColumn(client, 'users', 'email')).toBeNull() + }) + + it('returns null on a table with no EQL columns', async () => { + const { client } = mockClient(TABLE) + expect(await resolveEncryptedColumn(client, 'users', 'email')).toBeNull() }) }) diff --git a/packages/migrate/src/index.ts b/packages/migrate/src/index.ts index 79c400aa4..5adf7a5f0 100644 --- a/packages/migrate/src/index.ts +++ b/packages/migrate/src/index.ts @@ -1,15 +1,18 @@ /** * `@cipherstash/migrate` — primitives for migrating existing plaintext - * columns to `eql_v2_encrypted` in production Postgres databases. + * columns to EQL-encrypted columns (`eql_v2_encrypted` or the + * self-describing `eql_v3_*` domains) in production Postgres databases. * * Powers the `stash encrypt` CLI command group, and is usable directly * from a user's own worker/cron when they'd rather not pipe gigabytes * through a CLI process. * - * Per-column lifecycle: + * Per-column lifecycle (version-dependent — EQL v3 has no cut-over rename; + * the application switches to the encrypted column by name): * * ``` - * schema-added → dual-writing → backfilling → backfilled → cut-over → dropped + * v2: schema-added → dual-writing → backfilling → backfilled → cut-over → dropped + * v3: schema-added → dual-writing → backfilling → backfilled → dropped * ``` * * State is split across three stores on purpose: @@ -68,4 +71,11 @@ export { type MigrationStateRow, progress, } from './state.js' -export { detectColumnEqlVersion, type EqlVersion } from './version.js' +export { + classifyEqlDomain, + detectColumnEqlVersion, + type EncryptedColumnInfo, + type EqlVersion, + listEncryptedColumns, + resolveEncryptedColumn, +} from './version.js' diff --git a/packages/migrate/src/manifest.ts b/packages/migrate/src/manifest.ts index 943b2f463..872c83f54 100644 --- a/packages/migrate/src/manifest.ts +++ b/packages/migrate/src/manifest.ts @@ -60,13 +60,23 @@ const ManifestColumnSchema = z.object({ * CLI auto-detect via `information_schema`. */ pkColumn: z.string().optional(), + /** + * The encrypted counterpart column's name, recorded at backfill time. + * `_encrypted` is the CONVENTION but it is neither enforced nor + * relied upon — the EQL v3 domain types are self-describing, so commands + * resolve the encrypted column from the domain types + * (`resolveEncryptedColumn`) and use this field only as the first hint. + * Absent on manifests written before it existed. + */ + encryptedColumn: z.string().optional(), /** * The EQL version of the encrypted target column, recorded at backfill - * time from `detectColumnEqlVersion`. Lets `encrypt status`/`plan` - * branch without a DB round-trip: v3 columns have no - * `eql_v2_configuration` registration and no rename cut-over, so the - * v2-specific drift flags don't apply. Absent on manifests written - * before v3 support — those columns are necessarily v2. + * time from `detectColumnEqlVersion`. A cached HINT for display paths + * (`encrypt status` renders 'v3' and suppresses the v2-only drift flags): + * the source of truth is always the column's domain type in the database. + * Absent when the manifest predates v3 support OR when detection couldn't + * see the column at backfill time — readers must fall back to the domain + * type, never assume v2. */ eqlVersion: z.union([z.literal(2), z.literal(3)]).optional(), }) diff --git a/packages/migrate/src/version.ts b/packages/migrate/src/version.ts index 16ad7fd58..68ccf520e 100644 --- a/packages/migrate/src/version.ts +++ b/packages/migrate/src/version.ts @@ -6,43 +6,162 @@ import type { ClientBase } from 'pg' * (see {@link import('./eql.js')}), while v3 is domain-native — configuration * lives in the column's own type and there is no configuration table, so its * lifecycle is backfill-then-drop with no cut-over rename. + * + * Numeric (`2 | 3`) to match the manifest's `eqlVersion` field and the CLI + * installer's `--eql-version` — one representation everywhere, no + * string↔number translation at boundaries. */ -export type EqlVersion = 'v2' | 'v3' +export type EqlVersion = 2 | 3 + +/** An encrypted column found on a table, classified by its domain type. */ +export interface EncryptedColumnInfo { + /** The column's name, exactly as Postgres reports it. */ + column: string + /** The EQL domain name, e.g. `eql_v2_encrypted` or `eql_v3_text_search`. */ + domain: string + version: EqlVersion +} /** - * Detect the EQL version of a column by inspecting its Postgres type. + * Classify a Postgres domain-type name as an EQL generation. * - * - v2 encrypted columns are the `public.eql_v2_encrypted` domain. - * - v3 encrypted columns are a concrete `eql_v3_*` domain (e.g. - * `eql_v3_text_search`, `eql_v3_int8_ord`). - * - Anything else — a plaintext column, or a column/table that doesn't exist — - * returns `null`. + * EQL v3 types are deliberately self-describing — the domain name alone + * carries the generation — which is why this predicate is the ONE place the + * rule lives, and why detection never relies on column NAMES: the + * `_encrypted` naming is a convention, neither enforced nor required. + * + * - `eql_v2_encrypted` → 2 + * - `eql_v3_*` (e.g. `eql_v3_text_search`, `eql_v3_integer_ord`) → 3 + * - anything else → `null` (not an EQL column) + */ +export function classifyEqlDomain(domain: string): EqlVersion | null { + if (domain === 'eql_v2_encrypted') return 2 + if (domain.startsWith('eql_v3')) return 3 + return null +} + +/** + * Resolve `tableName` (optionally `schema.table`) to a regclass expression + * that preserves the identifier's case. * - * Pass the **encrypted target** column (e.g. `email_encrypted`), not the - * plaintext source: it's the encrypted column whose domain type carries the EQL - * generation. `tableName` may be schema-qualified (`"schema.table"`); - * resolution honours the connection's `search_path` via `to_regclass`. + * A bare `to_regclass($1)` PARSES its argument, case-folding unquoted names — + * so `to_regclass('User')` looks up `user` and misses a Prisma-style `"User"` + * table that the rest of the pipeline (which quotes identifiers verbatim, see + * `qualifyTable`/`quoteIdent` in cursor.ts) handles fine. `format('%I', …)` + * quotes the name first, making the lookup case-exact while still honouring + * `search_path` for unqualified names. + */ +const REGCLASS_SQL = `to_regclass( + CASE WHEN $2::text IS NULL THEN format('%I', $1::text) + ELSE format('%I.%I', $2::text, $1::text) END +)` + +/** Split `schema.table` the same way `qualifyTable` does (first dot wins). */ +function splitTableName(tableName: string): { + schema: string | null + table: string +} { + const dot = tableName.indexOf('.') + return dot >= 0 + ? { schema: tableName.slice(0, dot), table: tableName.slice(dot + 1) } + : { schema: null, table: tableName } +} + +/** + * Detect the EQL version of one named column by inspecting its Postgres + * domain type. Returns `null` for a plaintext column, a non-EQL domain, or a + * table/column that doesn't exist. + * + * `tableName` may be schema-qualified (`schema.table`); resolution is + * case-exact and honours `search_path` for unqualified names. */ export async function detectColumnEqlVersion( client: ClientBase, tableName: string, columnName: string, ): Promise { + const { schema, table } = splitTableName(tableName) // `a.atttypid` on a domain-typed column is the DOMAIN's oid, so `t.typname` // is the domain name (e.g. `eql_v2_encrypted`), not the underlying `jsonb`. - // `to_regclass` returns NULL for an unknown table → no rows → null. const result = await client.query<{ domain_name: string }>( `SELECT t.typname AS domain_name FROM pg_attribute a JOIN pg_type t ON t.oid = a.atttypid - WHERE a.attrelid = to_regclass($1) - AND a.attname = $2 + WHERE a.attrelid = ${REGCLASS_SQL} + AND a.attname = $3 AND NOT a.attisdropped`, - [tableName, columnName], + [table, schema, columnName], ) const domain = result.rows[0]?.domain_name - if (domain === undefined) return null - if (domain === 'eql_v2_encrypted') return 'v2' - if (domain.startsWith('eql_v3')) return 'v3' - return null + return domain === undefined ? null : classifyEqlDomain(domain) +} + +/** + * Every EQL-domain column on a table, classified. The EQL types are + * self-describing, so this is the ground truth for "which columns on this + * table are encrypted, and under which generation" — no naming convention + * involved. + */ +export async function listEncryptedColumns( + client: ClientBase, + tableName: string, +): Promise { + const { schema, table } = splitTableName(tableName) + const result = await client.query<{ column: string; domain_name: string }>( + `SELECT a.attname AS column, t.typname AS domain_name + FROM pg_attribute a + JOIN pg_type t ON t.oid = a.atttypid + WHERE a.attrelid = ${REGCLASS_SQL} + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum`, + [table, schema], + ) + const out: EncryptedColumnInfo[] = [] + for (const row of result.rows) { + const version = classifyEqlDomain(row.domain_name) + if (version !== null) { + out.push({ column: row.column, domain: row.domain_name, version }) + } + } + return out +} + +/** + * Find the encrypted counterpart of a plaintext column, trusting the domain + * types over any naming convention: + * + * 1. An explicit `hint` (from `--encrypted-column` or the manifest's recorded + * `encryptedColumn`) wins — but only if that column really carries an EQL + * domain. + * 2. Otherwise the `_encrypted` CONVENTION is tried — again validated + * against the domain type, never assumed. + * 3. Otherwise, if the table has exactly ONE EQL-domain column, that's the + * one — the self-describing types make the convention unnecessary. + * + * Returns `null` when nothing matches or when several EQL columns exist and + * none is identifiable (ambiguous — the caller should ask the user, listing + * `listEncryptedColumns` output). + */ +export async function resolveEncryptedColumn( + client: ClientBase, + tableName: string, + plaintextColumn: string, + hint?: string, +): Promise { + const candidates = await listEncryptedColumns(client, tableName) + if (candidates.length === 0) return null + + if (hint) { + return candidates.find((c) => c.column === hint) ?? null + } + + const conventional = candidates.find( + (c) => c.column === `${plaintextColumn}_encrypted`, + ) + if (conventional) return conventional + + // The plaintext column itself can't be its own encrypted counterpart. + const others = candidates.filter((c) => c.column !== plaintextColumn) + return others.length === 1 ? (others[0] ?? null) : null } diff --git a/skills/stash-cli/SKILL.md b/skills/stash-cli/SKILL.md index ffb1342ce..28d9405ae 100644 --- a/skills/stash-cli/SKILL.md +++ b/skills/stash-cli/SKILL.md @@ -424,7 +424,12 @@ For AI-guided integration that edits your existing schema files in place, prefer ### Encrypt -The cutover-step toolset: the database-side work that takes an existing plaintext column the rest of the way, **after** the rollout PR is deployed and dual-writes are live. It drives `@cipherstash/migrate`, recording every transition in `cipherstash.cs_migrations` (installed by `eql install`) and reading intent from `.cipherstash/migrations.json`. Internal phase names: `schema-added → dual-writing → backfilling → backfilled → cut-over → dropped`. +The database-side toolset that takes an existing plaintext column the rest of the way, **after** the rollout PR is deployed and dual-writes are live. It drives `@cipherstash/migrate`, recording every transition in `cipherstash.cs_migrations` (installed by `eql install`) and reading intent from `.cipherstash/migrations.json`. + +The phase ladder depends on the column's EQL version, which the commands detect from the column's **domain type** (EQL v3 types are self-describing; the `_encrypted` naming is a convention only, never relied upon): + +- **EQL v3 (the default):** `schema-added → dual-writing → backfilling → backfilled → dropped`. There is no cut-over — the application switches to the encrypted column by name, then the plaintext column is dropped. +- **EQL v2:** `schema-added → dual-writing → backfilling → backfilled → cut-over → dropped`, where cut-over renames the encrypted twin into the original column name. #### `encrypt status` / `encrypt plan` @@ -475,7 +480,7 @@ Flags: `--table`, `--column`, `--proxy-url `, `--migrations-dir `. stash encrypt drop --table users --column email ``` -Version-aware. For **EQL v2** columns in the `cut-over` phase it emits `ALTER TABLE
DROP COLUMN _plaintext;` (the post-rename name). For **EQL v3** columns it runs from the `backfilled` phase and drops the ORIGINAL `` — v3 has no rename, so make sure the application reads/writes `_encrypted` first. Either way it does **not** apply the migration — review and run your own migrate command. +Version-aware. For **EQL v2** columns in the `cut-over` phase it emits `ALTER TABLE
DROP COLUMN _plaintext;` (the post-rename name). For **EQL v3** columns it runs from the `backfilled` phase and drops the ORIGINAL `` — v3 has no rename, so make sure the application reads/writes the encrypted column first. Before generating, the v3 path **verifies live coverage** (refuses if any row still has `` set with the encrypted column NULL — e.g. rows written without dual-writes since the backfill; re-run `encrypt backfill` to fix). Either way it does **not** apply the migration — review and run your own migrate command. Flags: `--table`, `--column`, `--migrations-dir `. diff --git a/skills/stash-encryption/SKILL.md b/skills/stash-encryption/SKILL.md index a31e9f455..e6ea83813 100644 --- a/skills/stash-encryption/SKILL.md +++ b/skills/stash-encryption/SKILL.md @@ -796,11 +796,11 @@ Three sources of truth, kept separate on purpose: `stash encrypt status` shows all three side-by-side and flags drift (e.g. EQL says registered, the physical `_encrypted` column is missing). `stash status` (the quest log) rolls them up into the per-column "what's the next move" view used during a rollout. -> **Note on internal phase names.** The runtime event log uses `schema-added → dual-writing → backfilling → backfilled → cut-over → dropped` as machine-readable phase names. They appear in `cs_migrations` rows and `stash encrypt status` output. Treat them as internal mechanism detail — the user-facing story is "encryption rollout, then cutover, with a deploy gate in between." +> **Note on internal phase names.** The runtime event log uses machine-readable phase names that depend on the column's EQL version: v3 (the default) runs `schema-added → dual-writing → backfilling → backfilled → dropped` (no cut-over — the app switches to the encrypted column by name), while v2 runs `schema-added → dual-writing → backfilling → backfilled → cut-over → dropped`. They appear in `cs_migrations` rows and `stash encrypt status` output. Treat them as internal mechanism detail — the user-facing story is "encryption rollout, then switch reads to encrypted, with a deploy gate in between." ### CLI sequence for a single column -> **Known limitation:** `stash encrypt cutover` currently requires a pending EQL configuration registered via `stash db push`. SDK-only users may hit a "No pending EQL configuration" error. **Workaround:** Run `stash db push` once before `stash encrypt cutover`, even if you don't use CipherStash Proxy. Decoupling cutover from EQL config for SDK users is tracked separately. +> **Known limitation (v2):** `stash encrypt cutover` requires a pending EQL configuration registered via `stash db push`. SDK-only users may hit a "No pending EQL configuration" error. **Workaround:** Run `stash db push` once before `stash encrypt cutover`, even if you don't use CipherStash Proxy. Decoupling cutover from EQL config for SDK users is tracked separately. (EQL v3 columns never hit this — cutover doesn't apply to them.) ```bash # Run this often — it's the canonical "where am I?" command. From c04ffe3c0db8d681de9c9e1433bf42b15bc66ba0 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Thu, 16 Jul 2026 22:41:32 +1000 Subject: [PATCH 4/6] test(migrate): domain-typed target column; tighten eql_v3_ prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review-thread fixes on #649: the v3 backfill integration table now types email_encrypted with a domain carrying the real eql_v3_* storage CHECK shape (both scalar and SteVec arms), so the $N::jsonb write exercises the implicit jsonb→domain cast + CHECK enforcement instead of plain jsonb; and classifyEqlDomain matches the 'eql_v3_' prefix with the underscore so a hypothetical eql_v30_* generation can't classify as v3 (negative pins added). Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .../__tests__/backfill-v3.integration.test.ts | 28 +++++++++++++++---- .../migrate/src/__tests__/version.test.ts | 4 +++ packages/migrate/src/version.ts | 4 ++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/migrate/src/__tests__/backfill-v3.integration.test.ts b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts index aa61071bb..d4d7f274e 100644 --- a/packages/migrate/src/__tests__/backfill-v3.integration.test.ts +++ b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts @@ -6,7 +6,9 @@ * - the leak guard (`isEncryptedPayload`) accepts BOTH v3 wire shapes — flat * scalars (`{v:3, i, c}`) and SteVec documents (`{v:3, k:'sv', i, sv}`) — * so a v3 client's output flows through `runBackfill` unmodified; - * - the `$N::jsonb` write lands v3 envelopes in the target column; + * - the `$N::jsonb` write lands v3 envelopes in a DOMAIN-typed target column + * (implicit jsonb→domain cast + CHECK enforcement, the same assignment + * path a real `eql_v3_*` column takes); * - `countEncrypted` (the v3 verification primitive — v3 has no * `eql_v2.count_encrypted_with_active_config`) counts them. * @@ -19,9 +21,9 @@ * ``` * * No CipherStash credentials required — payloads are deterministic v3-shaped - * markers. End-to-end proof against a real `eql_v3_*` domain (whose CHECK - * constraint demands real ciphertext structure) lives with the live-crypto - * harness, not here. + * markers, and the target domain mirrors the real `eql_v3_*` storage CHECK + * (structure only; real-ciphertext proof lives with the live-crypto + * harness, not here). */ import 'dotenv/config' @@ -50,6 +52,22 @@ describe.skipIf(!runIntegration)('runBackfill with EQL v3 payloads', () => { await db.query('DROP SCHEMA IF EXISTS cipherstash CASCADE') await db.query('DROP SCHEMA IF EXISTS migrate_v3_test CASCADE') await db.query('CREATE SCHEMA migrate_v3_test') + // A domain with the SAME CHECK shape as the real `public.eql_v3_*` + // storage domains (object with v/i/c keys, v = 3), so the backfill's + // `$N::jsonb` write exercises the domain-typed assignment path — the + // implicit jsonb→domain cast plus CHECK enforcement — without needing + // an EQL install. A payload the real domain would reject fails here + // too (pinned below). + await db.query(` + CREATE DOMAIN migrate_v3_test.eql_v3_text_t AS jsonb + CHECK ( + jsonb_typeof(VALUE) = 'object' + AND VALUE ? 'v' + AND VALUE ? 'i' + AND VALUE->>'v' = '3' + AND (VALUE ? 'c' OR (VALUE->>'k' = 'sv' AND VALUE ? 'sv')) + ) + `) await installMigrationsSchema(db) } finally { db.release() @@ -107,7 +125,7 @@ describe.skipIf(!runIntegration)('runBackfill with EQL v3 payloads', () => { CREATE TABLE migrate_v3_test.users ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, email text NOT NULL, - email_encrypted jsonb + email_encrypted migrate_v3_test.eql_v3_text_t ) `) await db.query( diff --git a/packages/migrate/src/__tests__/version.test.ts b/packages/migrate/src/__tests__/version.test.ts index d0d9a7648..616cba8d5 100644 --- a/packages/migrate/src/__tests__/version.test.ts +++ b/packages/migrate/src/__tests__/version.test.ts @@ -32,6 +32,10 @@ describe('classifyEqlDomain', () => { expect(classifyEqlDomain('text')).toBeNull() expect(classifyEqlDomain('jsonb')).toBeNull() expect(classifyEqlDomain('citext')).toBeNull() + // Prefix is `eql_v3_` with the underscore — a hypothetical future + // `eql_v30_*` generation must not classify as v3. + expect(classifyEqlDomain('eql_v30_text')).toBeNull() + expect(classifyEqlDomain('eql_v3')).toBeNull() }) }) diff --git a/packages/migrate/src/version.ts b/packages/migrate/src/version.ts index 68ccf520e..6205dcdda 100644 --- a/packages/migrate/src/version.ts +++ b/packages/migrate/src/version.ts @@ -36,7 +36,9 @@ export interface EncryptedColumnInfo { */ export function classifyEqlDomain(domain: string): EqlVersion | null { if (domain === 'eql_v2_encrypted') return 2 - if (domain.startsWith('eql_v3')) return 3 + // Underscore included: a bare `startsWith('eql_v3')` would also claim + // hypothetical future generations like `eql_v30_*`. + if (domain.startsWith('eql_v3_')) return 3 return null } From 815341984fb1cecb7c6ddb82844bb25960383354 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Thu, 16 Jul 2026 22:42:43 +1000 Subject: [PATCH 5/6] =?UTF-8?q?docs(skills):=20cutover=20on=20v3=20is=20ph?= =?UTF-8?q?ase-aware=20=E2=80=94=20exit=200=20only=20when=20backfilled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- skills/stash-cli/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/stash-cli/SKILL.md b/skills/stash-cli/SKILL.md index 28d9405ae..8af0a830f 100644 --- a/skills/stash-cli/SKILL.md +++ b/skills/stash-cli/SKILL.md @@ -464,7 +464,7 @@ Backfill **auto-detects the target column's EQL version** from its Postgres doma stash encrypt cutover --table users --column email ``` -**EQL v2 only** — v3 has no cut-over: the application switches to `_encrypted` by name, and running this command on a v3 column reports "not applicable" (exit 0) with the next step. For v2, the preconditions are: the column is in the `backfilled` phase, **and** a pending EQL configuration exists. +**EQL v2 only** — v3 has no cut-over: the application switches to the encrypted column by name. Running this command on a **backfilled** v3 column reports "not applicable" (exit 0) with the next step; before backfill completes it exits 1 and says to finish the backfill (never "switch now" onto a half-populated column). For v2, the preconditions are: the column is in the `backfilled` phase, **and** a pending EQL configuration exists (on a v3-only database — no `eql_v2_configuration` table — it explains that and exits 1). In one transaction it renames `` → `_plaintext` and `_encrypted` → ``, advances the pending config to `encrypting`, activates it, and appends a `cut_over` event. With a Proxy URL configured (`--proxy-url` or `CIPHERSTASH_PROXY_URL`) it then calls `eql_v2.reload_config()` so Proxy picks up the new shape. From 605c40a4899e499374ac3308e172ed8ae4aef7a1 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Fri, 17 Jul 2026 10:58:21 +1000 Subject: [PATCH 6/6] fix(cli,migrate): harden EQL column resolution per #649 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups from Toby, James, and CodeRabbit: - Resolution provenance: pickEncryptedColumn (new pure core of resolveEncryptedColumn) tags every match `via: hint | convention | sole`. A sole-EQL-column match only proves uniqueness, not the plaintext<->ciphertext pairing, so `encrypt drop` — the one irreversible step — now refuses it with instructions instead of gating coverage on a possibly unrelated column. Display paths keep using it, so convention-free resolution still works everywhere else. - Fail closed on ambiguity: cutover and drop error listing the candidate EQL columns when none is identifiable, instead of falling through to the v2 machinery; the post-cutover v2 same-name state is recognised and still falls through to the v2 preconditions. - The generated v3 drop migration re-verifies coverage at APPLY time: LOCK TABLE, re-count plaintext-only rows, RAISE EXCEPTION without dropping if any appeared since generation; the DROP runs via EXECUTE in the same DO block so check-and-drop stay atomic under non-transactional runners. - resolveColumnLifecycle no longer swallows manifest read failures (ENOENT stays null via readManifest; malformed JSON/schema/permission errors propagate) and fetches the catalog once instead of up to three times. - cutover on an already-dropped v3 column says "lifecycle complete" (exit 0) instead of "finish the backfill". - stash status derives eqlVersion from the encrypted column's domain type (manifest encryptedColumn hint over the naming convention), falling back to the manifest only when the DB isn't visible — matching encrypt status. - Stale drop.ts JSDoc rewritten version-aware; migrate README fence language (MD040); stash-cli skill blockquote joined (MD028). Test-cast note: the `as unknown as ClientBase` factories in migrate tests stay (ClientBase's overloaded query signature can't be met by a structural fixture) but most resolution tests now target the pure pickEncryptedColumn and need no client at all. Claude-Session: https://claude.ai/code/session_01MxTTPaPP16m6br7Hoab94w --- .changeset/migrate-eql-v3.md | 22 ++- .../encrypt/__tests__/encrypt-v3.test.ts | 173 ++++++++++++++++-- packages/cli/src/commands/encrypt/cutover.ts | 28 ++- packages/cli/src/commands/encrypt/drop.ts | 111 +++++++++-- .../src/commands/encrypt/lib/resolve-eql.ts | 69 +++++-- packages/cli/src/commands/status/index.ts | 43 +++-- packages/cli/src/commands/status/quest.ts | 15 +- packages/migrate/README.md | 2 +- .../__tests__/backfill-v3.integration.test.ts | 3 + .../migrate/src/__tests__/version.test.ts | 111 ++++++----- packages/migrate/src/index.ts | 3 + packages/migrate/src/version.ts | 63 +++++-- skills/stash-cli/SKILL.md | 2 +- 13 files changed, 513 insertions(+), 132 deletions(-) diff --git a/.changeset/migrate-eql-v3.md b/.changeset/migrate-eql-v3.md index f504169b3..d1e055117 100644 --- a/.changeset/migrate-eql-v3.md +++ b/.changeset/migrate-eql-v3.md @@ -26,7 +26,15 @@ right lifecycle, no new flags: **verifies live coverage** (refuses to generate the migration while any row still has the plaintext set and the encrypted column NULL — the `countUnencrypted` check), and drops the ORIGINAL plaintext column (there - is no `_plaintext` under v3); v2 behaviour is unchanged. + is no `_plaintext` under v3); v2 behaviour is unchanged. The generated + v3 migration **re-verifies coverage at apply time** — it locks the table, + re-counts, and aborts without dropping if plaintext-only rows appeared + after generation. And because dropping is the one irreversible step, it + requires a positively asserted plaintext↔ciphertext pairing (the + manifest's recorded `encryptedColumn` or the naming convention): a match + found only by being the table's sole EQL column is refused with + instructions, and an ambiguous table (several EQL columns, none + identifiable) fails closed listing the candidates — as does `cutover`. - **`encrypt status`** classifies each column from the observed domain type (manifest as fallback), shows `v3` in the EQL column, and no longer raises the v2-only `not-registered` / `plaintext-col-missing` drift flags for v3 @@ -34,11 +42,13 @@ right lifecycle, no new flags: prompt teach the version-appropriate next step (no more "run cutover" on v3 columns). - New `@cipherstash/migrate` exports: `classifyEqlDomain`, - `resolveEncryptedColumn`, `listEncryptedColumns` (domain-type resolution — - case-exact for quoted/mixed-case table names), `countEncrypted` / - `countUnencrypted` (coverage counts), and manifest `eqlVersion` + - `encryptedColumn` fields. `EqlVersion` is numeric (`2 | 3`), matching the - manifest and the installer. + `resolveEncryptedColumn`, `pickEncryptedColumn`, `listEncryptedColumns` + (domain-type resolution — case-exact for quoted/mixed-case table names), + `countEncrypted` / `countUnencrypted` (coverage counts), and manifest + `eqlVersion` + `encryptedColumn` fields. `EqlVersion` is numeric (`2 | 3`), + matching the manifest and the installer. Resolved columns carry `via: + 'hint' | 'convention' | 'sole'` so callers can tell a positively asserted + pairing from a by-elimination guess. - Fixed: `encrypt cutover`/`encrypt drop` precondition failures now actually exit 1 — the early-return guards previously skipped the exit-code path entirely, so failed preconditions exited 0. (This also applies to v2 diff --git a/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts b/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts index b8e7c8eb5..da2389aff 100644 --- a/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts +++ b/packages/cli/src/commands/encrypt/__tests__/encrypt-v3.test.ts @@ -5,9 +5,10 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' // v2 config machine, and `encrypt drop` must target the ORIGINAL plaintext // column (there is no `_plaintext`) gated on `backfilled` AND a live // coverage check. Version + encrypted-column NAME come from the domain types -// via `resolveEncryptedColumn` — the `_encrypted` naming is a -// convention, never relied upon. The v2 paths are pinned alongside as -// regression guards. +// via `resolveColumnLifecycle` — the `_encrypted` naming is a +// convention, never relied upon — and both commands FAIL CLOSED when +// resolution is ambiguous instead of guessing a lifecycle. The v2 paths are +// pinned alongside as regression guards. const queryMock = vi.hoisted(() => vi.fn(async (_sql: string) => ({ rows: [] as unknown[] })), @@ -23,16 +24,12 @@ vi.mock('pg', () => ({ })) type ColumnInfo = { column: string; domain: string; version: 2 | 3 } +type ResolvedInfo = ColumnInfo & { via: 'hint' | 'convention' | 'sole' } +type Lifecycle = { info: ResolvedInfo | null; candidates: ColumnInfo[] } const migrateMocks = vi.hoisted(() => ({ - resolveEncryptedColumn: vi.fn( - async (): Promise => ({ - column: 'email_encrypted', - domain: 'eql_v3_text_search', - version: 3, - }), - ), listEncryptedColumns: vi.fn(async (): Promise => []), + pickEncryptedColumn: vi.fn(() => null), readManifest: vi.fn(async () => null), countUnencrypted: vi.fn(async () => 0), progress: vi.fn( @@ -47,6 +44,22 @@ const migrateMocks = vi.hoisted(() => ({ })) vi.mock('@cipherstash/migrate', () => migrateMocks) +// Mock the lifecycle RESOLUTION (each test states its scenario directly) +// but keep the real `explainUnresolved` — the fail-closed messaging is part +// of what these tests pin. +const lifecycleMock = vi.hoisted(() => + vi.fn( + async (): Promise<{ + info: (ColumnInfo & { via: string }) | null + candidates: ColumnInfo[] + }> => ({ info: null, candidates: [] }), + ), +) +vi.mock('../lib/resolve-eql.js', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, resolveColumnLifecycle: lifecycleMock } +}) + vi.mock('@clack/prompts', () => ({ intro: vi.fn(), outro: vi.fn(), @@ -104,6 +117,13 @@ const V2_INFO: ColumnInfo = { version: 2, } +function resolved( + info: ColumnInfo, + via: ResolvedInfo['via'] = 'convention', +): Lifecycle { + return { info: { ...info, via }, candidates: [info] } +} + /** v2 config machine present + a pending config row. */ function mockV2ConfigQueries() { queryMock.mockImplementation(async (sql: string) => { @@ -129,7 +149,7 @@ describe('encrypt cutover — EQL version awareness', () => { vi.clearAllMocks() queryMock.mockResolvedValue({ rows: [] }) migrateMocks.progress.mockResolvedValue({ phase: 'backfilled' }) - migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_INFO) + lifecycleMock.mockResolvedValue(resolved(V3_INFO)) }) it('short-circuits on a backfilled v3 column: guidance, no rename, no config machine, exit 0', async () => { @@ -169,8 +189,22 @@ describe('encrypt cutover — EQL version awareness', () => { exitSpy.mockRestore() }) + it('v3 already dropped: terminal phase is "nothing to do", not "finish the backfill"; exit 0', async () => { + migrateMocks.progress.mockResolvedValue({ phase: 'dropped' }) + const exitSpy = spyExit() + + await cutoverCommand({ table: 'users', column: 'email' }) + + expect(p.log.info).toHaveBeenCalledWith( + expect.stringContaining('already completed'), + ) + expect(p.log.error).not.toHaveBeenCalled() + expect(exitSpy).not.toHaveBeenCalled() + exitSpy.mockRestore() + }) + it('uses the RESOLVED encrypted column name, not the naming convention', async () => { - migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_CUSTOM_INFO) + lifecycleMock.mockResolvedValue(resolved(V3_CUSTOM_INFO, 'hint')) await cutoverCommand({ table: 'users', column: 'email' }) @@ -182,8 +216,29 @@ describe('encrypt cutover — EQL version awareness', () => { ) }) + it('fails closed when EQL columns exist but none is identifiable', async () => { + lifecycleMock.mockResolvedValue({ + info: null, + candidates: [ + { column: 'a_enc', domain: 'eql_v3_text_eq', version: 3 }, + { column: 'b_enc', domain: 'eql_v3_text_eq', version: 3 }, + ], + }) + const exitSpy = spyExit() + + await cutoverCommand({ table: 'users', column: 'email' }) + + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining('Cannot identify which encrypted column'), + ) + expect(p.log.error).toHaveBeenCalledWith(expect.stringContaining('a_enc')) + expect(migrateMocks.renameEncryptedColumns).not.toHaveBeenCalled() + expect(exitSpy).toHaveBeenCalledWith(1) + exitSpy.mockRestore() + }) + it('still runs the v2 flow for a v2 column (regression pin)', async () => { - migrateMocks.resolveEncryptedColumn.mockResolvedValue(V2_INFO) + lifecycleMock.mockResolvedValue(resolved(V2_INFO)) mockV2ConfigQueries() await cutoverCommand({ table: 'users', column: 'email' }) @@ -196,8 +251,7 @@ describe('encrypt cutover — EQL version awareness', () => { it('explains a v3-only database instead of a raw relation-does-not-exist error', async () => { // Detection missed (e.g. no EQL columns visible) → v2 path — but the // config table doesn't exist on this database. - migrateMocks.resolveEncryptedColumn.mockResolvedValue(null) - migrateMocks.listEncryptedColumns.mockResolvedValue([]) + lifecycleMock.mockResolvedValue({ info: null, candidates: [] }) queryMock.mockImplementation(async (sql: string) => typeof sql === 'string' && sql.includes('to_regclass') ? { rows: [{ exists: null }] } @@ -220,7 +274,7 @@ describe('encrypt drop — EQL version awareness', () => { beforeEach(() => { vi.clearAllMocks() queryMock.mockResolvedValue({ rows: [] }) - migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_INFO) + lifecycleMock.mockResolvedValue(resolved(V3_INFO)) migrateMocks.countUnencrypted.mockResolvedValue(0) }) @@ -246,6 +300,24 @@ describe('encrypt drop — EQL version awareness', () => { ) }) + it('v3: the generated migration re-verifies coverage at APPLY time, atomically', async () => { + migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilled' }) + + await dropCommand({ table: 'users', column: 'email' }) + + // The CLI-side count goes stale the moment the file is written; the + // migration must lock, re-count, and abort without dropping if any + // plaintext-only row appeared between generation and application. + const sql = writeFileMock.mock.calls[0]?.[1] as string + expect(sql).toContain('LOCK TABLE "users" IN ACCESS EXCLUSIVE MODE') + expect(sql).toContain('RAISE EXCEPTION') + expect(sql).toContain('"email" IS NOT NULL') + expect(sql).toContain('"email_encrypted" IS NULL') + // The DROP itself runs inside the same DO block (EXECUTE), so + // check-and-drop stay atomic even under non-transactional runners. + expect(sql).toMatch(/EXECUTE 'ALTER TABLE "users" DROP COLUMN "email"'/) + }) + it('v3: refuses to generate when rows are still plaintext-only', async () => { migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilled' }) migrateMocks.countUnencrypted.mockResolvedValueOnce(7) @@ -266,7 +338,7 @@ describe('encrypt drop — EQL version awareness', () => { }) it('v3: gates coverage on the RESOLVED encrypted column name', async () => { - migrateMocks.resolveEncryptedColumn.mockResolvedValue(V3_CUSTOM_INFO) + lifecycleMock.mockResolvedValue(resolved(V3_CUSTOM_INFO, 'hint')) migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilled' }) await dropCommand({ table: 'users', column: 'email' }) @@ -281,6 +353,54 @@ describe('encrypt drop — EQL version awareness', () => { expect(sql).toContain('DROP COLUMN "email"') }) + it("v3: refuses a by-elimination ('sole') match — uniqueness cannot prove the pairing", async () => { + // The table's ONE EQL column may encrypt a DIFFERENT field; gating + // coverage on it and dropping `email` could destroy the only copy. + lifecycleMock.mockResolvedValue( + resolved( + { column: 'secret_blob', domain: 'eql_v3_text_search', version: 3 }, + 'sole', + ), + ) + // No progress stub: the sole-match guard fires before the phase gate + // ever consults cs_migrations. (Queuing an unconsumed + // mockResolvedValueOnce here would leak into the next test.) + const exitSpy = spyExit() + + await dropCommand({ table: 'users', column: 'email' }) + + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining('nothing confirms it encrypts "email"'), + ) + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining('--encrypted-column secret_blob'), + ) + expect(migrateMocks.countUnencrypted).not.toHaveBeenCalled() + expect(writeFileMock).not.toHaveBeenCalled() + expect(exitSpy).toHaveBeenCalledWith(1) + exitSpy.mockRestore() + }) + + it('fails closed when EQL columns exist but none is identifiable', async () => { + lifecycleMock.mockResolvedValue({ + info: null, + candidates: [ + { column: 'a_enc', domain: 'eql_v3_text_eq', version: 3 }, + { column: 'b_enc', domain: 'eql_v3_text_eq', version: 3 }, + ], + }) + const exitSpy = spyExit() + + await dropCommand({ table: 'users', column: 'email' }) + + expect(p.log.error).toHaveBeenCalledWith( + expect.stringContaining('Cannot identify which encrypted column'), + ) + expect(writeFileMock).not.toHaveBeenCalled() + expect(exitSpy).toHaveBeenCalledWith(1) + exitSpy.mockRestore() + }) + it('v3: rejects when not yet backfilled', async () => { migrateMocks.progress.mockResolvedValueOnce({ phase: 'backfilling' }) const exitSpy = spyExit() @@ -296,7 +416,24 @@ describe('encrypt drop — EQL version awareness', () => { }) it('v2: unchanged — requires cut-over, no coverage gate, drops _plaintext (regression pin)', async () => { - migrateMocks.resolveEncryptedColumn.mockResolvedValue(V2_INFO) + lifecycleMock.mockResolvedValue(resolved(V2_INFO)) + migrateMocks.progress.mockResolvedValueOnce({ phase: 'cut-over' }) + + await dropCommand({ table: 'users', column: 'email' }) + + const sql = writeFileMock.mock.calls[0]?.[1] as string + expect(sql).toContain('DROP COLUMN "email_plaintext"') + expect(migrateMocks.countUnencrypted).not.toHaveBeenCalled() + }) + + it('v2 post-cutover: `email` itself carrying the v2 domain is NOT ambiguity — proceeds down the v2 path', async () => { + // After cutover renamed the ciphertext onto `email`, no counterpart is + // resolvable BY DESIGN. The fail-closed guard must recognize this state + // rather than blocking the one drop the lifecycle actually wants. + lifecycleMock.mockResolvedValue({ + info: null, + candidates: [{ column: 'email', domain: 'eql_v2_encrypted', version: 2 }], + }) migrateMocks.progress.mockResolvedValueOnce({ phase: 'cut-over' }) await dropCommand({ table: 'users', column: 'email' }) diff --git a/packages/cli/src/commands/encrypt/cutover.ts b/packages/cli/src/commands/encrypt/cutover.ts index ca9624dca..6584ef075 100644 --- a/packages/cli/src/commands/encrypt/cutover.ts +++ b/packages/cli/src/commands/encrypt/cutover.ts @@ -12,7 +12,7 @@ import { detectDrizzle } from '@/commands/db/detect.js' import { detectPackageManager, runnerCommand } from '@/commands/init/utils.js' import { loadStashConfig } from '@/config/index.js' import { scaffoldDrizzleMigration } from './drizzle-helper.js' -import { resolveColumnLifecycle } from './lib/resolve-eql.js' +import { explainUnresolved, resolveColumnLifecycle } from './lib/resolve-eql.js' /** * Options accepted by `stash encrypt cutover`. Swaps the plaintext and @@ -70,15 +70,39 @@ export async function cutoverCommand(options: CutoverCommandOptions) { // DOMAIN TYPES (manifest name as a hint; the `_encrypted` naming is // a convention, never relied upon) before any phase/config checks so v3 // users get the real answer, not a confusing precondition error. - const { info } = await resolveColumnLifecycle( + const { info, candidates } = await resolveColumnLifecycle( client, options.table, options.column, ) + // Fail closed on ambiguity: `info === null` with EQL columns present + // means we can't tell WHICH lifecycle applies — running the v2 config + // machine against (possibly) v3 columns would only produce a misleading + // downstream error. (No EQL columns at all, or the post-cutover v2 + // same-name state, still falls through to the v2 preconditions below.) + const unresolved = explainUnresolved( + options.table, + options.column, + candidates, + ) + if (!info && unresolved) { + p.log.error(unresolved) + exitCode = 1 + return + } const state = await progress(client, options.table, options.column) if (info?.version === 3) { const encryptedColumn = info.column + if (state?.phase === 'dropped') { + // Terminal phase — the lifecycle already finished. Not an error and + // not "finish the backfill": there is nothing left to backfill. + p.log.info( + `${options.table}.${options.column} has already completed the EQL v3 lifecycle (plaintext dropped). Nothing to cut over.`, + ) + p.outro('Nothing to do for EQL v3.') + return + } if (state?.phase !== 'backfilled') { // Not a "nothing to do" — the user isn't ready for ANY next step // yet. Exit 1 so scripted pipelines gating on cutover don't read diff --git a/packages/cli/src/commands/encrypt/drop.ts b/packages/cli/src/commands/encrypt/drop.ts index 321bd8662..2fe440bb0 100644 --- a/packages/cli/src/commands/encrypt/drop.ts +++ b/packages/cli/src/commands/encrypt/drop.ts @@ -12,20 +12,24 @@ import { detectDrizzle } from '@/commands/db/detect.js' import { detectPackageManager, runnerCommand } from '@/commands/init/utils.js' import { loadStashConfig } from '@/config/index.js' import { scaffoldDrizzleMigration } from './drizzle-helper.js' -import { resolveColumnLifecycle } from './lib/resolve-eql.js' +import { explainUnresolved, resolveColumnLifecycle } from './lib/resolve-eql.js' /** - * Options accepted by `stash encrypt drop`. Generates a migration file - * that drops the now-unused plaintext column (renamed to `_plaintext` - * by cutover). Does *not* apply the migration — the user runs their usual - * migration tool (drizzle-kit, prisma, psql) to actually execute it. + * Options accepted by `stash encrypt drop`. Generates a migration file that + * drops the now-unused plaintext column — for EQL v2 that is + * `_plaintext` (the name cutover's rename left it with); for EQL v3 + * (no rename) it is the original `` itself. Does *not* apply the + * migration — the user runs their usual migration tool (drizzle-kit, + * prisma, psql) to actually execute it. */ export interface DropCommandOptions { /** Physical table name, e.g. `users`. */ table: string /** - * Physical column — the original plaintext name. The generated migration - * drops `_plaintext` (the name the column has *after* cutover). + * Physical column — the original plaintext name. What gets dropped + * depends on the column's EQL version: v2 drops `_plaintext` + * (post-cutover leftover); v3 drops `` itself, gated on a live + * ciphertext-coverage check. */ column: string /** @@ -37,8 +41,11 @@ export interface DropCommandOptions { } /** - * CLI handler for `stash encrypt drop`. Requires the column to be in - * phase `cut-over`; otherwise errors out. + * CLI handler for `stash encrypt drop`. Version-aware preconditions: + * EQL v2 requires phase `cut-over`; EQL v3 (which has no cut-over) requires + * `backfilled` plus a live coverage check — and the generated v3 migration + * re-verifies coverage at APPLY time, since rows can be written between + * generation and application. * * For Drizzle projects, scaffolds the migration via `drizzle-kit generate * --custom` so the file lands with the correct sequential prefix and a @@ -67,11 +74,41 @@ export async function dropCommand(options: DropCommandOptions) { // column, droppable straight after `backfilled`. The version and the // encrypted column's name are resolved from the DOMAIN TYPES (manifest // name as a hint) — the `_encrypted` naming is a convention only. - const { info } = await resolveColumnLifecycle( + const { info, candidates } = await resolveColumnLifecycle( client, options.table, options.column, ) + // Fail closed on ambiguity: with EQL columns present but no identifiable + // counterpart, guessing a lifecycle here could validate coverage against + // the wrong ciphertext and generate an irreversible drop of the wrong + // data. (The post-cutover v2 state — `` itself carries the v2 + // domain, counterpart legitimately unresolvable — falls through to the + // v2 path; live truth from the DB wins over the manifest's cached + // version throughout.) + const unresolved = explainUnresolved( + options.table, + options.column, + candidates, + ) + if (!info && unresolved) { + p.log.error(unresolved) + exitCode = 1 + return + } + // A `via: 'sole'` match only proves the column is the table's ONE EQL + // column — it may encrypt a DIFFERENT field, in which case the coverage + // gate below would count the wrong ciphertext and wave through a drop + // that destroys the only copy of this data. Dropping is the single + // irreversible step in the lifecycle, so it demands a positively + // asserted pairing (manifest hint or the naming convention). + if (info?.via === 'sole') { + p.log.error( + `${options.table}.${info.column} (${info.domain}) is the table's only encrypted column, but nothing confirms it encrypts "${options.column}" — refusing to generate an irreversible drop on that guess. Record the pairing and retry: re-run \`stash encrypt backfill --table ${options.table} --column ${options.column} --encrypted-column ${info.column}\` (which writes it to the manifest), or set "encryptedColumn": "${info.column}" for this column in .cipherstash/migrations.json.`, + ) + exitCode = 1 + return + } const isV3 = info?.version === 3 const encryptedColumn = info?.column ?? `${options.column}_encrypted` const requiredPhase = isV3 ? 'backfilled' : 'cut-over' @@ -120,7 +157,9 @@ export async function dropCommand(options: DropCommandOptions) { dot >= 0 ? `"${options.table.slice(0, dot)}"."${options.table.slice(dot + 1)}"` : `"${options.table}"` - const dropSql = `-- Generated by stash encrypt drop\n-- Drops the plaintext column now that ${options.table}.${isV3 ? encryptedColumn : options.column} is encrypted.\n\nALTER TABLE ${qualifiedTable} DROP COLUMN "${plaintextToDrop}";\n` + const dropSql = isV3 + ? buildV3DropSql(qualifiedTable, options.column, encryptedColumn) + : `-- Generated by stash encrypt drop\n-- Drops the plaintext column now that ${options.table}.${options.column} is encrypted.\n\nALTER TABLE ${qualifiedTable} DROP COLUMN "${plaintextToDrop}";\n` const cwd = process.cwd() const migrationsDir = options.migrationsDir ?? 'drizzle' @@ -185,3 +224,53 @@ export async function dropCommand(options: DropCommandOptions) { if (exitCode) process.exit(exitCode) } } + +/** + * Build the v3 drop migration. The CLI's coverage check above goes stale + * the moment the file is written — rows can be inserted plaintext-only + * between generation and application (a bulk import, a service that isn't + * dual-writing). So the migration re-verifies coverage at APPLY time, + * atomically: it takes the same ACCESS EXCLUSIVE lock the DROP COLUMN + * needs (blocking concurrent writes for the remainder of the transaction), + * re-counts, and aborts the whole migration — dropping nothing — if any + * plaintext-only row appeared. The DROP runs via EXECUTE inside the same + * DO block so check-and-drop stay atomic even under migration runners that + * don't wrap files in a transaction (plain `psql -f`). + * + * Identifiers arrive pre-validated (they resolved against the live catalog + * above); embedded string literals escape single quotes defensively. + */ +function buildV3DropSql( + qualifiedTable: string, + plaintextColumn: string, + encryptedColumn: string, +): string { + const lit = (s: string) => s.replace(/'/g, "''") + return `-- Generated by stash encrypt drop +-- Drops the plaintext column now that ${qualifiedTable}."${encryptedColumn}" is encrypted. +-- Coverage is re-verified here, at apply time: the check stash ran at +-- generation time cannot see rows written after it. + +DO $stash_drop$ +DECLARE + unencrypted bigint; +BEGIN + -- The lock DROP COLUMN takes anyway, acquired up front so the + -- count-then-drop below is atomic against concurrent writes. + LOCK TABLE ${qualifiedTable} IN ACCESS EXCLUSIVE MODE; + + SELECT count(*) INTO unencrypted + FROM ${qualifiedTable} + WHERE "${plaintextColumn}" IS NOT NULL + AND "${encryptedColumn}" IS NULL; + + IF unencrypted > 0 THEN + RAISE EXCEPTION 'stash encrypt drop: refusing to drop %.% — % row(s) have plaintext set but % NULL. Dropping now would permanently destroy that data. Re-run: stash encrypt backfill, then regenerate this migration.', + '${lit(qualifiedTable)}', '${lit(plaintextColumn)}', unencrypted, '${lit(encryptedColumn)}'; + END IF; + + EXECUTE 'ALTER TABLE ${lit(qualifiedTable)} DROP COLUMN "${lit(plaintextColumn)}"'; +END +$stash_drop$; +` +} diff --git a/packages/cli/src/commands/encrypt/lib/resolve-eql.ts b/packages/cli/src/commands/encrypt/lib/resolve-eql.ts index 505df38a9..9ce9afb6a 100644 --- a/packages/cli/src/commands/encrypt/lib/resolve-eql.ts +++ b/packages/cli/src/commands/encrypt/lib/resolve-eql.ts @@ -1,22 +1,26 @@ import { type EncryptedColumnInfo, listEncryptedColumns, + pickEncryptedColumn, + type ResolvedEncryptedColumn, readManifest, - resolveEncryptedColumn, } from '@cipherstash/migrate' import type pg from 'pg' /** * The resolved encryption lifecycle for one plaintext column: which column - * carries the ciphertext, and which EQL generation its domain type declares. + * carries the ciphertext, which EQL generation its domain type declares, + * and which rule identified it (`info.via` — destructive commands must not + * act on a `'sole'` match; see {@link pickEncryptedColumn}). */ export interface ResolvedLifecycle { /** The encrypted counterpart, or `null` when none could be identified. */ - info: EncryptedColumnInfo | null + info: ResolvedEncryptedColumn | null /** - * Every EQL-domain column on the table — non-empty when resolution failed - * because several candidates exist and none is identifiable. Lets the - * caller name them instead of erroring blind. + * Every EQL-domain column on the table, always populated (it's the same + * catalog read resolution picks from). When `info` is `null` and this is + * non-empty, resolution failed because none of these candidates is + * identifiable — callers name them instead of erroring blind. */ candidates: EncryptedColumnInfo[] } @@ -31,28 +35,61 @@ export interface ResolvedLifecycle { * backfill`, including any `--encrypted-column` override) — used as a * HINT and still validated against the actual domain type. * 2. The `_encrypted` convention, validated the same way. - * 3. The table's sole EQL-domain column, if there is exactly one. + * 3. The table's sole EQL-domain column, if there is exactly one — flagged + * `via: 'sole'` because uniqueness cannot prove the pairing. * - * The VERSION always comes from the domain type — the manifest's cached - * `eqlVersion` is for display paths that have no DB connection. + * The VERSION always comes from the domain type. This deliberately diverges + * from `encrypt status`, which falls back to the manifest's cached + * `eqlVersion`: status may run without DB access, whereas the lifecycle + * commands always hold a connection, so live truth wins here. + * + * A missing manifest is fine (`readManifest` returns `null` on ENOENT) but + * any other manifest failure — malformed JSON, schema mismatch, permissions + * — propagates: the callers are destructive commands, and silently losing + * the recorded column hint must not let them fall through to a guess. */ export async function resolveColumnLifecycle( client: pg.ClientBase, table: string, column: string, ): Promise { - const manifest = await readManifest().catch(() => null) + const manifest = await readManifest() const hint = manifest?.tables[table]?.find( (entry) => entry.column === column, )?.encryptedColumn - let info = hint - ? await resolveEncryptedColumn(client, table, column, hint) - : null + const candidates = await listEncryptedColumns(client, table) + let info = hint ? pickEncryptedColumn(candidates, column, hint) : null // A stale hint (column since renamed/retyped) must not mask a resolvable // counterpart — fall back to convention + sole-EQL-column resolution. - if (!info) info = await resolveEncryptedColumn(client, table, column) - if (info) return { info, candidates: [] } + if (!info) info = pickEncryptedColumn(candidates, column) + return { info, candidates } +} - return { info: null, candidates: await listEncryptedColumns(client, table) } +/** + * Explain a failed resolution (`info === null`) to the user, or return + * `null` when the failure is fine to fall through to the v2 lifecycle: + * + * - No EQL columns at all → the v2 phase/config preconditions produce the + * accurate error ("not backfilled", "no pending config", …). + * - The plaintext column ITSELF carries the v2 domain → the normal + * post-cutover v2 state (`` was renamed onto the ciphertext), where + * "no counterpart" is expected, not a problem. + * + * Anything else means EQL columns exist but none is identifiable — the + * caller must fail closed with this message rather than guess a lifecycle. + */ +export function explainUnresolved( + table: string, + column: string, + candidates: readonly EncryptedColumnInfo[], +): string | null { + if (candidates.length === 0) return null + if (candidates.some((c) => c.column === column && c.version === 2)) { + return null + } + const listed = candidates + .map((c) => ` - ${c.column} (${c.domain})`) + .join('\n') + return `Cannot identify which encrypted column corresponds to ${table}.${column}. EQL columns on ${table}:\n${listed}\nRecord the pairing and retry: re-run \`stash encrypt backfill --table ${table} --column ${column} --encrypted-column \` (which writes it to the manifest), or set "encryptedColumn" for this column in .cipherstash/migrations.json.` } diff --git a/packages/cli/src/commands/status/index.ts b/packages/cli/src/commands/status/index.ts index 08063a426..a3449ff20 100644 --- a/packages/cli/src/commands/status/index.ts +++ b/packages/cli/src/commands/status/index.ts @@ -1,6 +1,10 @@ import { existsSync } from 'node:fs' import { resolve } from 'node:path' -import { type Manifest, readManifest } from '@cipherstash/migrate' +import { + classifyEqlDomain, + type Manifest, + readManifest, +} from '@cipherstash/migrate' import * as p from '@clack/prompts' import pg from 'pg' import { @@ -31,18 +35,27 @@ import { * timeout (~75s on most platforms). */ const CONNECT_TIMEOUT_MS = 2_000 -function manifestColumns( - manifest: Manifest, -): { table: string; column: string; eqlVersion?: 2 | 3 }[] { - const out: { table: string; column: string; eqlVersion?: 2 | 3 }[] = [] +interface ManifestColumn { + table: string + column: string + eqlVersion?: 2 | 3 + encryptedColumn?: string +} + +function manifestColumns(manifest: Manifest): ManifestColumn[] { + const out: ManifestColumn[] = [] for (const [table, cols] of Object.entries(manifest.tables)) { for (const col of cols) { out.push({ table, column: col.column, - // Cached hint only — the quest ladder shape (4 rungs for v3, 5 for - // v2) is display, so the manifest's record is good enough here. + // Cached hints only — when the DB is reachable, the encrypted + // column's domain type wins (see gatherObservations); these cover + // the DB-unreachable render. ...(col.eqlVersion ? { eqlVersion: col.eqlVersion } : {}), + ...(col.encryptedColumn + ? { encryptedColumn: col.encryptedColumn } + : {}), }) } } @@ -118,15 +131,23 @@ export async function gatherObservations( const key: `${string}.${string}` = `${c.table}.${c.column}` const phaseRow = phases.get(key) const eqlInfo = eqlConfig.get(key) + // Version resolution mirrors `encrypt status` (renderRow): the + // encrypted column's DOMAIN TYPE is self-describing and wins; the + // manifest's cached eqlVersion covers the window where the physical + // column isn't visible. The recorded encryptedColumn is preferred + // over the `_encrypted` naming convention. + const cols = physicalCols.get(c.table) + const encryptedName = c.encryptedColumn ?? `${c.column}_encrypted` + const domain = cols?.get(encryptedName) + const eqlVersion = + (domain ? classifyEqlDomain(domain) : null) ?? c.eqlVersion return { table: c.table, column: c.column, - ...(c.eqlVersion ? { eqlVersion: c.eqlVersion } : {}), + ...(eqlVersion ? { eqlVersion } : {}), phase: phaseRow ? phaseRow.phase : null, eql: eqlInfo ? { state: eqlInfo.state } : null, - physicalEncryptedTwinExists: ( - physicalCols.get(c.table) ?? new Map() - ).has(`${c.column}_encrypted`), + physicalEncryptedTwinExists: cols?.has(encryptedName) ?? false, } }) diff --git a/packages/cli/src/commands/status/quest.ts b/packages/cli/src/commands/status/quest.ts index b4225e4ef..2c5ceca3d 100644 --- a/packages/cli/src/commands/status/quest.ts +++ b/packages/cli/src/commands/status/quest.ts @@ -70,15 +70,18 @@ export interface ColumnObservation { /** From `eql_v2_configuration`. `null` means not registered. `undefined` * means DB unreachable. */ eql?: EqlColumnSummary | null - /** Whether `_encrypted` exists in `information_schema.columns`. - * Used as a fallback signal that schema-add has been applied even if + /** Whether the encrypted counterpart (the manifest's recorded + * `encryptedColumn`, falling back to the `_encrypted` + * convention) exists in `information_schema.columns`. Used as a + * fallback signal that schema-add has been applied even if * cs_migrations doesn't yet track this column. `undefined` when the * caller can't tell. */ physicalEncryptedTwinExists?: boolean - /** The column's EQL generation, from the manifest's cached `eqlVersion`. - * v3 has a 4-objective ladder (no cut-over rename — the app switches to - * the encrypted column by name). `undefined` (unknown / pre-v3 manifest) - * renders the v2 ladder. */ + /** The column's EQL generation — from the encrypted column's domain type + * when the DB is reachable (self-describing), else the manifest's cached + * `eqlVersion`. v3 has a 4-objective ladder (no cut-over rename — the + * app switches to the encrypted column by name). `undefined` (unknown / + * pre-v3 manifest) renders the v2 ladder. */ eqlVersion?: 2 | 3 } diff --git a/packages/migrate/README.md b/packages/migrate/README.md index 8a05e22ea..ea1fb6549 100644 --- a/packages/migrate/README.md +++ b/packages/migrate/README.md @@ -8,7 +8,7 @@ Backs the `stash encrypt` CLI command group, but also exported for direct use Each column walks through these phases — the ladder depends on the column's EQL version (auto-detected from its Postgres domain type via `detectColumnEqlVersion`): -``` +```text EQL v2: schema-added → dual-writing → backfilling → backfilled → cut-over → dropped EQL v3: schema-added → dual-writing → backfilling → backfilled ————————————→ dropped ``` diff --git a/packages/migrate/src/__tests__/backfill-v3.integration.test.ts b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts index d4d7f274e..bbd73f31e 100644 --- a/packages/migrate/src/__tests__/backfill-v3.integration.test.ts +++ b/packages/migrate/src/__tests__/backfill-v3.integration.test.ts @@ -294,6 +294,9 @@ describe.skipIf(!runIntegration)( ) expect(info?.column).toBe('secret_blob') expect(info?.version).toBe(3) + // By-elimination provenance: display paths may use this match, but + // `encrypt drop` refuses it — uniqueness cannot prove the pairing. + expect(info?.via).toBe('sole') }) }, ) diff --git a/packages/migrate/src/__tests__/version.test.ts b/packages/migrate/src/__tests__/version.test.ts index 616cba8d5..b6cfdb9d6 100644 --- a/packages/migrate/src/__tests__/version.test.ts +++ b/packages/migrate/src/__tests__/version.test.ts @@ -3,10 +3,15 @@ import { describe, expect, it, vi } from 'vitest' import { classifyEqlDomain, detectColumnEqlVersion, + type EncryptedColumnInfo, listEncryptedColumns, + pickEncryptedColumn, resolveEncryptedColumn, } from '../version.js' +// The one contained type-erasing cast in this file: the functions under +// test take a pg.ClientBase but only ever call `.query`, and ClientBase's +// overloaded query signature can't be satisfied by a structural fixture. function mockClient(rows: Array>) { const query = vi.fn().mockResolvedValue({ rows }) return { client: { query } as unknown as ClientBase, query } @@ -97,69 +102,87 @@ describe('listEncryptedColumns', () => { }) }) -describe('resolveEncryptedColumn', () => { - const TABLE = [ - { column: 'id', domain_name: 'int8' }, - { column: 'email', domain_name: 'text' }, - ] - - it('an explicit hint wins, validated against the domain type', async () => { - const { client } = mockClient([ - ...TABLE, - { column: 'email_enc', domain_name: 'eql_v3_text_search' }, - { column: 'email_encrypted', domain_name: 'eql_v3_text_eq' }, - ]) - expect( - await resolveEncryptedColumn(client, 'users', 'email', 'email_enc'), - ).toEqual({ column: 'email_enc', domain: 'eql_v3_text_search', version: 3 }) +describe('pickEncryptedColumn', () => { + const col = ( + column: string, + domain = 'eql_v3_text_eq', + version: 2 | 3 = 3, + ): EncryptedColumnInfo => ({ column, domain, version }) + + it('an explicit hint wins, validated against the domain type', () => { + const candidates = [ + col('email_enc', 'eql_v3_text_search'), + col('email_encrypted'), + ] + expect(pickEncryptedColumn(candidates, 'email', 'email_enc')).toEqual({ + column: 'email_enc', + domain: 'eql_v3_text_search', + version: 3, + via: 'hint', + }) }) - it('a hint naming a non-EQL column resolves to null, not a guess', async () => { - const { client } = mockClient([ - ...TABLE, - { column: 'email_encrypted', domain_name: 'eql_v3_text_eq' }, - ]) + it('a hint naming a non-EQL column resolves to null, not a guess', () => { expect( - await resolveEncryptedColumn(client, 'users', 'email', 'email'), + pickEncryptedColumn([col('email_encrypted')], 'email', 'email'), ).toBeNull() }) - it('falls back to the _encrypted convention', async () => { - const { client } = mockClient([ - ...TABLE, - { column: 'email_encrypted', domain_name: 'eql_v3_text_eq' }, - { column: 'other_encrypted', domain_name: 'eql_v3_text_eq' }, - ]) - expect(await resolveEncryptedColumn(client, 'users', 'email')).toEqual({ + it('falls back to the _encrypted convention', () => { + const candidates = [col('email_encrypted'), col('other_encrypted')] + expect(pickEncryptedColumn(candidates, 'email')).toEqual({ column: 'email_encrypted', domain: 'eql_v3_text_eq', version: 3, + via: 'convention', }) }) - it('resolves a sole EQL column regardless of its name — the convention is never required', async () => { - const { client } = mockClient([ - ...TABLE, - { column: 'secret_blob', domain_name: 'eql_v3_text_search' }, - ]) - expect(await resolveEncryptedColumn(client, 'users', 'email')).toEqual({ + it("resolves a sole EQL column regardless of its name — flagged 'sole', because uniqueness cannot prove the pairing", () => { + // The convention is never REQUIRED (the whole point of self-describing + // v3 types), but a by-elimination match may encrypt a different field — + // `via: 'sole'` is what lets destructive callers refuse to act on it. + expect( + pickEncryptedColumn([col('secret_blob', 'eql_v3_text_search')], 'email'), + ).toEqual({ column: 'secret_blob', domain: 'eql_v3_text_search', version: 3, + via: 'sole', }) }) - it('returns null when several EQL columns exist and none is identifiable', async () => { - const { client } = mockClient([ - ...TABLE, - { column: 'a_enc', domain_name: 'eql_v3_text_eq' }, - { column: 'b_enc', domain_name: 'eql_v3_text_eq' }, - ]) - expect(await resolveEncryptedColumn(client, 'users', 'email')).toBeNull() + it('never resolves the plaintext column to itself', () => { + // Post-cutover v2: `email` itself carries the v2 domain. It is the + // ciphertext, not a counterpart of itself. + expect( + pickEncryptedColumn([col('email', 'eql_v2_encrypted', 2)], 'email'), + ).toBeNull() }) - it('returns null on a table with no EQL columns', async () => { - const { client } = mockClient(TABLE) - expect(await resolveEncryptedColumn(client, 'users', 'email')).toBeNull() + it('returns null when several EQL columns exist and none is identifiable', () => { + expect( + pickEncryptedColumn([col('a_enc'), col('b_enc')], 'email'), + ).toBeNull() + }) + + it('returns null with no EQL columns', () => { + expect(pickEncryptedColumn([], 'email')).toBeNull() + }) +}) + +describe('resolveEncryptedColumn', () => { + it('picks from the live catalog (fetch + pick passthrough)', async () => { + const { client } = mockClient([ + { column: 'id', domain_name: 'int8' }, + { column: 'email', domain_name: 'text' }, + { column: 'email_encrypted', domain_name: 'eql_v3_text_eq' }, + ]) + expect(await resolveEncryptedColumn(client, 'users', 'email')).toEqual({ + column: 'email_encrypted', + domain: 'eql_v3_text_eq', + version: 3, + via: 'convention', + }) }) }) diff --git a/packages/migrate/src/index.ts b/packages/migrate/src/index.ts index 5adf7a5f0..65bd0db15 100644 --- a/packages/migrate/src/index.ts +++ b/packages/migrate/src/index.ts @@ -75,7 +75,10 @@ export { classifyEqlDomain, detectColumnEqlVersion, type EncryptedColumnInfo, + type EncryptedColumnResolution, type EqlVersion, listEncryptedColumns, + pickEncryptedColumn, + type ResolvedEncryptedColumn, resolveEncryptedColumn, } from './version.js' diff --git a/packages/migrate/src/version.ts b/packages/migrate/src/version.ts index 6205dcdda..dccfe7e35 100644 --- a/packages/migrate/src/version.ts +++ b/packages/migrate/src/version.ts @@ -130,40 +130,71 @@ export async function listEncryptedColumns( } /** - * Find the encrypted counterpart of a plaintext column, trusting the domain - * types over any naming convention: + * Which rule identified the encrypted counterpart. Callers gate on this: + * `hint` and `convention` positively assert the plaintext↔ciphertext + * pairing; `sole` only proves the column is the table's ONE EQL column — + * it may encrypt a *different* field, so irreversible operations (dropping + * the plaintext) must not act on it without explicit confirmation. + */ +export type EncryptedColumnResolution = 'hint' | 'convention' | 'sole' + +/** An {@link EncryptedColumnInfo} plus how it was identified. */ +export interface ResolvedEncryptedColumn extends EncryptedColumnInfo { + via: EncryptedColumnResolution +} + +/** + * Pick the encrypted counterpart of a plaintext column from an already + * fetched candidate list (see {@link listEncryptedColumns}), trusting the + * domain types over any naming convention: * * 1. An explicit `hint` (from `--encrypted-column` or the manifest's recorded * `encryptedColumn`) wins — but only if that column really carries an EQL - * domain. + * domain (`via: 'hint'`). * 2. Otherwise the `_encrypted` CONVENTION is tried — again validated - * against the domain type, never assumed. + * against the domain type, never assumed (`via: 'convention'`). * 3. Otherwise, if the table has exactly ONE EQL-domain column, that's the - * one — the self-describing types make the convention unnecessary. + * best guess (`via: 'sole'`) — the self-describing types make the + * convention unnecessary, but uniqueness alone cannot prove the pairing; + * check `via` before doing anything destructive. * - * Returns `null` when nothing matches or when several EQL columns exist and - * none is identifiable (ambiguous — the caller should ask the user, listing - * `listEncryptedColumns` output). + * Pure — callers with several lookups against the same table fetch the + * candidates once and pick repeatedly. Returns `null` when nothing matches + * or when several EQL columns exist and none is identifiable (ambiguous — + * the caller should ask the user, listing the candidates). */ -export async function resolveEncryptedColumn( - client: ClientBase, - tableName: string, +export function pickEncryptedColumn( + candidates: readonly EncryptedColumnInfo[], plaintextColumn: string, hint?: string, -): Promise { - const candidates = await listEncryptedColumns(client, tableName) +): ResolvedEncryptedColumn | null { if (candidates.length === 0) return null if (hint) { - return candidates.find((c) => c.column === hint) ?? null + const hinted = candidates.find((c) => c.column === hint) + return hinted ? { ...hinted, via: 'hint' } : null } const conventional = candidates.find( (c) => c.column === `${plaintextColumn}_encrypted`, ) - if (conventional) return conventional + if (conventional) return { ...conventional, via: 'convention' } // The plaintext column itself can't be its own encrypted counterpart. const others = candidates.filter((c) => c.column !== plaintextColumn) - return others.length === 1 ? (others[0] ?? null) : null + return others.length === 1 && others[0] ? { ...others[0], via: 'sole' } : null +} + +/** + * {@link pickEncryptedColumn} over a live catalog read — fetches the + * table's EQL-domain columns and picks from them. + */ +export async function resolveEncryptedColumn( + client: ClientBase, + tableName: string, + plaintextColumn: string, + hint?: string, +): Promise { + const candidates = await listEncryptedColumns(client, tableName) + return pickEncryptedColumn(candidates, plaintextColumn, hint) } diff --git a/skills/stash-cli/SKILL.md b/skills/stash-cli/SKILL.md index 8af0a830f..de04c8d37 100644 --- a/skills/stash-cli/SKILL.md +++ b/skills/stash-cli/SKILL.md @@ -469,7 +469,7 @@ stash encrypt cutover --table users --column email In one transaction it renames `` → `_plaintext` and `_encrypted` → ``, advances the pending config to `encrypting`, activates it, and appends a `cut_over` event. With a Proxy URL configured (`--proxy-url` or `CIPHERSTASH_PROXY_URL`) it then calls `eql_v2.reload_config()` so Proxy picks up the new shape. > **After cutover, `` holds ciphertext — the read path is not automatic.** Wire reads through the encryption client (`decryptModel(row, table)` for Drizzle, the `encryptedSupabaseV3` wrapper for Supabase — `encryptedSupabase` on the legacy v2 surface, otherwise `decrypt` / `bulkDecryptModels`) before returning values to callers. Skip this and your read paths hand raw EQL payloads to end users. The integration skill has the exact API. **CipherStash Proxy is the one exception** — it decrypts on the wire, so Proxy users need no application change. The cutover plan written by `stash plan` includes this read-path switch as an explicit step. - +> > **Known gap (v2).** The pending-configuration precondition is satisfied by `stash db push`. SDK-only users (who otherwise never need `db push`) must therefore run it once before `encrypt cutover`. (EQL v3 columns sidestep this entirely — no configuration table, no cutover; see above.) Tracked in [issue #585](https://github.com/cipherstash/stack/issues/585). Flags: `--table`, `--column`, `--proxy-url `, `--migrations-dir `.