diff --git a/apps/alerting/src/worker.test.ts b/apps/alerting/src/worker.test.ts index 588d1759e..212905d61 100644 --- a/apps/alerting/src/worker.test.ts +++ b/apps/alerting/src/worker.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "@effect/vitest" -import { Cause, Effect, Exit, Layer } from "effect" +import { Cause, Effect, Exit, Fiber, Latch, Layer } from "effect" import { env as workerEnv } from "../test/stubs/cloudflare-workers" import { buildLayer, catchTickFailure, selectScheduledProgram, type ScheduledTickPrograms } from "./worker" @@ -39,6 +39,37 @@ describe("alerting Effect root", () => { }) } + it.effect("never starts fixVerification before the error tick completes", () => { + // The adversarial schedule for concurrency 2: alert and escalation are + // instant, so both finish while error is still running. Listing the four + // ticks side by side would hand the freed slot to fixVerification here — + // only the explicit error→fixVerification chain keeps the ordering. + const calls: Array = [] + const record = (name: string) => Effect.sync(() => calls.push(name)).pipe(Effect.asVoid) + return Effect.gen(function* () { + const errorGate = yield* Latch.make(false) + const ticks = { + alert: record("alert"), + anomaly: record("anomaly"), + cloudflareAnalytics: record("cloudflareAnalytics"), + digest: record("digest"), + error: errorGate.await.pipe(Effect.andThen(record("error"))), + escalation: record("escalation"), + fixVerification: record("fixVerification"), + planetScale: record("planetScale"), + serviceMapRollup: record("serviceMapRollup"), + } satisfies ScheduledTickPrograms + const fiber = yield* Effect.forkChild(selectScheduledProgram("* * * * *", ticks)) + // Let alert and escalation run to completion while error is held open. + yield* Effect.yieldNow + yield* Effect.yieldNow + expect(calls).not.toContain("fixVerification") + yield* errorGate.open + yield* Fiber.join(fiber) + expect(calls.indexOf("error")).toBeLessThan(calls.indexOf("fixVerification")) + }) + }) + it.effect("fails closed for an unknown cron", () => { const calls: Array = [] const tick = (name: string) => Effect.sync(() => calls.push(name)).pipe(Effect.asVoid) diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index 0b91bfc76..1c24a6044 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -461,10 +461,12 @@ export const selectScheduledProgram = ( Match.when("*/15 * * * *", () => ticks.digest), Match.when("0 * * * *", () => ticks.serviceMapRollup), Match.when("* * * * *", () => - // `fixVerification` runs after `error` in the same group so a window that - // this minute's error tick just refuted is already settled when the - // verification tick looks at it — one fewer wasted agent start. - Effect.all([ticks.alert, ticks.error, ticks.escalation, ticks.fixVerification], { + // `fixVerification` is chained onto `error` rather than listed beside it: + // a window this minute's error tick just refuted must already be settled + // when the verification tick looks at it, and array order under bounded + // concurrency does not promise that — alert and escalation finishing + // first would have started fixVerification while error still ran. + Effect.all([ticks.alert, Effect.andThen(ticks.error, ticks.fixVerification), ticks.escalation], { concurrency: 2, discard: true, }), diff --git a/apps/api/src/chat/loop/turn.test.ts b/apps/api/src/chat/loop/turn.test.ts index 4354d53a8..f2a6f0567 100644 --- a/apps/api/src/chat/loop/turn.test.ts +++ b/apps/api/src/chat/loop/turn.test.ts @@ -1113,3 +1113,57 @@ describe("runChatTurn closing submit", () => { }), ) }) + +describe("runChatTurn duplicate completion calls", () => { + it.live("dispatches only the first of duplicate completion calls in an ordinary step", () => + Effect.gen(function* () { + // The completion is an exactly-once output channel: a response that + // carries it twice must not execute two competing submits. + const record: Array = [] + const result = yield* collect( + [ + [ + toolCall("c1", SUBMIT, { claim: "first" }), + toolCall("c2", SUBMIT, { claim: "second" }), + finish("tool-calls"), + ], + ], + { completion: submitCompletion(record) }, + ) + + assert.deepEqual(record, [{ claim: "first" }]) + const submitEvents = result.events.filter( + (event) => event.type === "tool-call" && event.name === SUBMIT, + ) + assert.lengthOf(submitEvents, 1) + const end = terminal(result.events)[0] + assert.equal(end?.type === "turn-end" ? end.reason : undefined, "stop") + }), + ) + + it.live("dispatches only the first duplicate on the forced closing step", () => + Effect.gen(function* () { + const record: Array = [] + // Prose first step forces the submit closing step; the model then + // ignores "exactly one call" and emits the completion twice. + const result = yield* collect( + [ + [textDelta("thinking..."), finish()], + [ + toolCall("c1", SUBMIT, { claim: "first" }), + toolCall("c2", SUBMIT, { claim: "second" }), + finish("tool-calls"), + ], + ], + { completion: submitCompletion(record) }, + ) + + assert.deepEqual(record, [{ claim: "first" }]) + const submitEvents = result.events.filter( + (event) => event.type === "tool-call" && event.name === SUBMIT, + ) + assert.lengthOf(submitEvents, 1) + assert.lengthOf(terminal(result.events), 1) + }), + ) +}) diff --git a/apps/api/src/chat/loop/turn.ts b/apps/api/src/chat/loop/turn.ts index 31938468a..0c129c29b 100644 --- a/apps/api/src/chat/loop/turn.ts +++ b/apps/api/src/chat/loop/turn.ts @@ -477,9 +477,20 @@ const runStep = ( }) } + // The completion is an exactly-once output channel: a response carrying + // duplicate completion calls must not run it twice (competing submits + // would leave the report reflecting one payload and its side effects + // another). Keep the first and drop the rest before any dispatch. + let completionSeen = false const calls = response.events .filter(LLMEvent.is.toolCall) .filter((call) => !call.providerExecuted) + .filter((call) => { + if (call.name !== input.completion?.name) return true + if (completionSeen) return false + completionSeen = true + return true + }) const finishReason = response.finishReason?.normalized const providerFailure = response.events.find(LLMEvent.is.providerError) diff --git a/apps/api/src/platform/Apns.test.ts b/apps/api/src/platform/Apns.test.ts new file mode 100644 index 000000000..a97cfa31e --- /dev/null +++ b/apps/api/src/platform/Apns.test.ts @@ -0,0 +1,45 @@ +import { assert, describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { makeSingleFlightTokenCache } from "./Apns" + +describe("makeSingleFlightTokenCache", () => { + it.live("many concurrent reads on a cold cache mint exactly once", () => + Effect.gen(function* () { + let mints = 0 + // The mint suspends (as the real ECDSA sign does), so without + // single-flight every concurrent fiber would observe the empty cache + // and mint its own token. + const mint = Effect.gen(function* () { + mints += 1 + yield* Effect.promise(() => new Promise((resolve) => setTimeout(resolve, 5))) + return `token-${mints}` + }) + const currentToken = yield* makeSingleFlightTokenCache(60_000, mint) + const tokens = yield* Effect.all( + Array.from({ length: 8 }, () => currentToken), + { concurrency: "unbounded" }, + ) + assert.strictEqual(mints, 1) + assert.deepStrictEqual( + tokens, + Array.from({ length: 8 }, () => "token-1"), + ) + }), + ) + + it.live("mints again only after the TTL expires", () => + Effect.gen(function* () { + let mints = 0 + const mint = Effect.sync(() => { + mints += 1 + return `token-${mints}` + }) + const currentToken = yield* makeSingleFlightTokenCache(10, mint) + assert.strictEqual(yield* currentToken, "token-1") + assert.strictEqual(yield* currentToken, "token-1") + yield* Effect.promise(() => new Promise((resolve) => setTimeout(resolve, 15))) + assert.strictEqual(yield* currentToken, "token-2") + assert.strictEqual(mints, 2) + }), + ) +}) diff --git a/apps/api/src/platform/Apns.ts b/apps/api/src/platform/Apns.ts index 8699e08bd..0b6078290 100644 --- a/apps/api/src/platform/Apns.ts +++ b/apps/api/src/platform/Apns.ts @@ -1,4 +1,4 @@ -import { Clock, Context, Effect, Layer, Option, Redacted, Ref, Schema } from "effect" +import { Clock, Context, Effect, Layer, Option, Redacted, Ref, Schema, Semaphore } from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" import type { MobilePushEnvironment } from "@maple/domain/http" import { Env, type EnvConfig } from "./Env" @@ -198,6 +198,48 @@ const resolveConfig = (env: EnvConfig): ApnsConfig | null => { const decodeReason = Schema.decodeUnknownOption(Schema.Struct({ reason: Schema.String })) +/** + * TTL cache around `mint` with single-flight refresh. MobilePushService fans + * out up to eight concurrent sends, so a cold or expired cache would otherwise + * mint one JWT per fiber — and Apple throttles clients that update provider + * tokens too often (`TooManyProviderTokenUpdates`). One permit serializes the + * stale path; waiters re-check the cache and reuse the winner's token. + * Exported for the concurrency test only. + */ +export const makeSingleFlightTokenCache = ( + ttlMs: number, + mint: Effect.Effect, +): Effect.Effect> => + Ref.make(Option.none<{ readonly value: string; readonly mintedAtMs: number }>()).pipe( + Effect.map((cached) => { + const lock = Semaphore.makeUnsafe(1) + // A cached token counts only while younger than the TTL; expiry is + // absence, not a null to compare against. + const freshValue = (nowMs: number) => + Ref.get(cached).pipe( + Effect.map(Option.filter((entry) => nowMs - entry.mintedAtMs < ttlMs)), + Effect.map(Option.map((entry) => entry.value)), + ) + return Effect.gen(function* () { + const nowMs = yield* Clock.currentTimeMillis + const entry = yield* freshValue(nowMs) + if (Option.isSome(entry)) return entry.value + return yield* lock.withPermits(1)( + Effect.gen(function* () { + // Double-checked: a fiber that waited here usually finds the + // winner's fresh token and must not mint another. + const innerNowMs = yield* Clock.currentTimeMillis + const latest = yield* freshValue(innerNowMs) + if (Option.isSome(latest)) return latest.value + const value = yield* mint + yield* Ref.set(cached, Option.some({ value, mintedAtMs: innerNowMs })) + return value + }), + ) + }) + }), + ) + export class ApnsClient extends Context.Service()( "@maple/api/platform/ApnsClient", { @@ -237,11 +279,6 @@ export class ApnsClient extends Context.Service()( }), ) - const cachedToken = yield* Ref.make<{ - readonly value: string - readonly mintedAtMs: number - } | null>(null) - const mintToken = Effect.fn("ApnsClient.mintToken")(function* () { const nowMs = yield* Clock.currentTimeMillis const header = base64UrlString(JSON.stringify({ alg: "ES256", kid: config.keyId })) @@ -260,17 +297,10 @@ export class ApnsClient extends Context.Service()( ), catch: (cause) => new ApnsError({ message: "APNs JWT signing failed", cause }), }) - const value = `${signingInput}.${base64UrlBytes(signature)}` - yield* Ref.set(cachedToken, { value, mintedAtMs: nowMs }) - return value + return `${signingInput}.${base64UrlBytes(signature)}` }) - const currentToken = Effect.gen(function* () { - const nowMs = yield* Clock.currentTimeMillis - const cached = yield* Ref.get(cachedToken) - if (cached !== null && nowMs - cached.mintedAtMs < TOKEN_TTL_MS) return cached.value - return yield* mintToken() - }) + const currentToken = yield* makeSingleFlightTokenCache(TOKEN_TTL_MS, mintToken()) const send = Effect.fn("ApnsClient.send")(function* (push: ApnsPush) { yield* Effect.annotateCurrentSpan({ diff --git a/apps/api/src/services/alerts/AlertDeliveryDispatch.test.ts b/apps/api/src/services/alerts/AlertDeliveryDispatch.test.ts index ec55546b4..1d045958e 100644 --- a/apps/api/src/services/alerts/AlertDeliveryDispatch.test.ts +++ b/apps/api/src/services/alerts/AlertDeliveryDispatch.test.ts @@ -1,5 +1,6 @@ import type { AlertDestinationRow } from "@maple/db" import { + AlertDeliveryAuthError, AlertDeliveryError, AlertDeliveryRejectedError, AlertDeliveryTargetMissingError, @@ -569,6 +570,136 @@ describe("dispatchDelivery", () => { }), ) + it.effect("slack-bot: the delivery timeout aborts the underlying request", () => + Effect.gen(function* () { + // A timeout that leaves the POST running is a duplicate page in waiting: + // the queue retries while the "timed-out" request still delivers. + let sawSignal = false + let aborted = false + const fetchFn: typeof fetch = (_input, init) => + new Promise((_resolve, reject) => { + sawSignal = init?.signal != null + init?.signal?.addEventListener("abort", () => { + aborted = true + reject(new DOMException("The operation was aborted", "AbortError")) + }) + }) + + const fiber = yield* Effect.forkChild( + Effect.flip( + dispatchDelivery(slackBotContext, "{}", fetchFn, 5_000, LINK, CHAT, slackTokenDeps()), + ), + { startImmediately: true }, + ) + yield* TestClock.adjust("6 seconds") + const error = yield* Fiber.join(fiber) + + assert.instanceOf(error, AlertDeliveryError) + assert.include(error.message, "timed out") + assert.isTrue(sawSignal) + assert.isTrue(aborted) + }), + ) + + it.effect("webhook: the abort signal survives the SSRF-guarded fetch path", () => + Effect.gen(function* () { + const webhookContext: DispatchContext = { + ...pagerdutyContext, + destination: { ...destinationRow, name: "Webhook", type: "webhook" }, + publicConfig: { summary: "POST hooks.example.test", channelLabel: null }, + secretConfig: { + type: "webhook", + url: "https://hooks.example.test/maple", + signingSecret: null, + }, + } + let aborted = false + const fetchFn: typeof fetch = (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + aborted = true + reject(new DOMException("The operation was aborted", "AbortError")) + }) + }) + + const fiber = yield* Effect.forkChild( + Effect.flip(dispatchDelivery(webhookContext, "{}", fetchFn, 5_000, LINK, CHAT, noEmailDeps)), + { startImmediately: true }, + ) + yield* TestClock.adjust("6 seconds") + const error = yield* Fiber.join(fiber) + + assert.instanceOf(error, AlertDeliveryError) + assert.include(error.message, "timed out") + assert.isTrue(aborted) + }), + ) + + it.effect("slack-bot: an auth error code is terminal, not retried forever", () => + Effect.gen(function* () { + const fetchFn: typeof fetch = async () => + new Response(JSON.stringify({ ok: false, error: "invalid_auth" }), { status: 200 }) + + const error = yield* Effect.flip( + dispatchDelivery(slackBotContext, "{}", fetchFn, 5_000, LINK, CHAT, slackTokenDeps()), + ) + + // Slack reports auth failure as HTTP 200 + `ok:false`. Retrying replays + // the same dead token; only classifying it terminal lets the failure + // streak disable the destination and surface it in the setup audit. + assert.instanceOf(error, AlertDeliveryAuthError) + assert.isFalse(error.error.retryable) + assert.strictEqual(error.providerErrorCode, "invalid_auth") + }), + ) + + it.effect("slack-bot: an archived channel is a missing target", () => + Effect.gen(function* () { + const fetchFn: typeof fetch = async () => + new Response(JSON.stringify({ ok: false, error: "is_archived" }), { status: 200 }) + + const error = yield* Effect.flip( + dispatchDelivery(slackBotContext, "{}", fetchFn, 5_000, LINK, CHAT, slackTokenDeps()), + ) + + assert.instanceOf(error, AlertDeliveryTargetMissingError) + assert.isFalse(error.error.retryable) + assert.strictEqual(error.providerErrorCode, "is_archived") + }), + ) + + it.effect("slack-bot: a permanently rejected payload is not retryable", () => + Effect.gen(function* () { + const fetchFn: typeof fetch = async () => + new Response(JSON.stringify({ ok: false, error: "invalid_blocks" }), { status: 200 }) + + const error = yield* Effect.flip( + dispatchDelivery(slackBotContext, "{}", fetchFn, 5_000, LINK, CHAT, slackTokenDeps()), + ) + + assert.instanceOf(error, AlertDeliveryRejectedError) + assert.isFalse(error.error.retryable) + assert.strictEqual(error.providerErrorCode, "invalid_blocks") + }), + ) + + it.effect("slack-bot: an unrecognized error code stays retryable", () => + Effect.gen(function* () { + const fetchFn: typeof fetch = async () => + new Response(JSON.stringify({ ok: false, error: "fatal_error" }), { status: 200 }) + + const error = yield* Effect.flip( + dispatchDelivery(slackBotContext, "{}", fetchFn, 5_000, LINK, CHAT, slackTokenDeps()), + ) + + // Mis-classifying transient as terminal costs a destination its + // enablement; unknown codes err on the retry side. + assert.instanceOf(error, AlertDeliveryError) + assert.isTrue(error.error.retryable) + assert.strictEqual(error.providerErrorCode, "fatal_error") + }), + ) + it.effect("slack-bot: a non-JSON 200 response fails typed", () => Effect.gen(function* () { const fetchFn: typeof fetch = async () => new Response("gateway says hi", { status: 200 }) diff --git a/apps/api/src/services/alerts/AlertDestinationsService.ts b/apps/api/src/services/alerts/AlertDestinationsService.ts index 81edf87ad..37cce5749 100644 --- a/apps/api/src/services/alerts/AlertDestinationsService.ts +++ b/apps/api/src/services/alerts/AlertDestinationsService.ts @@ -23,7 +23,7 @@ import { type UserId, } from "@maple/domain/http" import { alertDestinations, alertRules, type AlertDestinationRow } from "@maple/db" -import { and, desc, eq } from "drizzle-orm" +import { and, desc, eq, sql } from "drizzle-orm" import { Context, Effect, Layer, Match, Option, Redacted, Schema } from "effect" import { encryptAes256Gcm, type EncryptedValue } from "@/platform/Crypto" import { Database } from "@/platform/DatabaseLive" @@ -852,13 +852,47 @@ export class AlertDestinationsService extends Context.Service< }), ) } - const deleted = yield* dbExecute((db) => - db - .delete(alertDestinations) - .where(and(eq(alertDestinations.orgId, orgId), eq(alertDestinations.id, destinationId))) - .returning(txidColumn), + // The scan above is the rich-message UX path; this transaction is the + // correctness gate. Rule writes validate destinations under the same + // per-org advisory lock, so re-checking references inside it closes the + // race where a rule commits a reference between our scan and the delete. + const deleteResult = yield* dbExecute((db) => + db.transaction(async (tx) => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) + const stillReferenced = await tx + .select({ id: alertRules.id, name: alertRules.name }) + .from(alertRules) + .where( + and( + eq(alertRules.orgId, orgId), + sql`${alertRules.destinationIdsJson} @> ${JSON.stringify([destinationId])}::jsonb`, + ), + ) + if (stillReferenced.length > 0) { + return { referencedBy: stillReferenced, deleted: [] } + } + const deleted = await tx + .delete(alertDestinations) + .where( + and(eq(alertDestinations.orgId, orgId), eq(alertDestinations.id, destinationId)), + ) + .returning(txidColumn) + return { referencedBy: [], deleted } + }), ) - const txid = readTxid(deleted) + if (deleteResult.referencedBy.length > 0) { + return yield* Effect.fail( + new AlertDestinationInUseError({ + message: `Destination is still used by alert rules: ${deleteResult.referencedBy + .map((rule) => rule.name) + .join(", ")}`, + destinationId, + ruleIds: deleteResult.referencedBy.map((rule) => decodeAlertRuleIdSync(rule.id)), + ruleNames: deleteResult.referencedBy.map((rule) => rule.name), + }), + ) + } + const txid = readTxid(deleteResult.deleted) return new AlertDestinationDeleteResponse({ id: destinationId, ...(txid !== undefined ? { txid } : undefined), diff --git a/apps/api/src/services/alerts/AlertRulesService.ts b/apps/api/src/services/alerts/AlertRulesService.ts index 3182be659..75a2f0b3b 100644 --- a/apps/api/src/services/alerts/AlertRulesService.ts +++ b/apps/api/src/services/alerts/AlertRulesService.ts @@ -24,7 +24,7 @@ import { alertRuleStates, } from "@maple/db" import { and, desc, eq, inArray, sql } from "drizzle-orm" -import { Array as Arr, Context, Effect, HashSet, Layer, Schema } from "effect" +import { Array as Arr, Context, Effect, HashSet, Layer, Match, Schema } from "effect" import { Database, type DatabaseApi } from "@/platform/DatabaseLive" import { makeDbExecute } from "@/platform/db-execute" import { readTxid, txidColumn } from "@/platform/electric-txid" @@ -152,7 +152,6 @@ export const makeAlertRulePersistence = (options: { request: AlertRuleUpsertRequest, ) { const normalized = yield* normalizeRule(orgId, request) - yield* requireDestinationIds(orgId, normalized.destinationIds) const ruleId = existingId ?? normalized.id const timestamp = yield* runtime.now const ruleFields = { @@ -191,6 +190,26 @@ export const makeAlertRulePersistence = (options: { const writeResult = yield* dbExecute((db) => db.transaction(async (tx) => { await tx.execute(sql`select pg_advisory_xact_lock(hashtext(${orgId}))`) + // Destination existence is checked INSIDE the lock: destination + // deletion takes the same per-org advisory lock around its reference + // scan, so a rule can no longer commit a reference to a destination + // whose deletion validated "unreferenced" concurrently. + if (normalized.destinationIds.length > 0) { + const destinationRows = await tx + .select({ id: alertDestinations.id }) + .from(alertDestinations) + .where( + and( + eq(alertDestinations.orgId, orgId), + inArray(alertDestinations.id, [...normalized.destinationIds]), + ), + ) + const existingIds = new Set(destinationRows.map((destination) => destination.id)) + const missingDestinationId = normalized.destinationIds.find((id) => !existingIds.has(id)) + if (missingDestinationId !== undefined) { + return { _tag: "MissingDestination" as const, destinationId: missingDestinationId } + } + } if (normalized.enabled) { const activeRows = await tx .select({ id: alertRules.id }) @@ -199,7 +218,7 @@ export const makeAlertRulePersistence = (options: { const alreadyActive = existingId != null && activeRows.some((row) => row.id === existingId) if (!alreadyActive && activeRows.length >= MAX_ACTIVE_ALERT_RULES_PER_ORG) { - return { limitExceeded: true as const, writeRows: [] } + return { _tag: "LimitExceeded" as const } } } @@ -220,22 +239,38 @@ export const makeAlertRulePersistence = (options: { .set(ruleFields) .where(and(eq(alertRules.orgId, orgId), eq(alertRules.id, existingId))) .returning(txidColumn) - return { limitExceeded: false as const, writeRows } + return { _tag: "Written" as const, writeRows } }), ) - if (writeResult.limitExceeded) { - return yield* Effect.fail( - makeAlertValidationError( - `Organizations may have at most ${MAX_ACTIVE_ALERT_RULES_PER_ORG} active alert rules`, + // The transaction runs in Promise-land, so it reports its outcome as a tagged + // value and the failures are raised out here — `Match.exhaustive` is what makes + // a fourth outcome a compile error rather than a silently ignored branch. + return yield* Match.value(writeResult).pipe( + Match.tag("MissingDestination", (outcome) => + Effect.fail( + new AlertRuleDestinationNotFoundError({ + message: "Alert rule references an unknown destination", + destinationId: outcome.destinationId, + }), ), - ) - } - return { - normalized, - ruleId, - timestamp, - txid: readTxid(writeResult.writeRows), - } + ), + Match.tag("LimitExceeded", () => + Effect.fail( + makeAlertValidationError( + `Organizations may have at most ${MAX_ACTIVE_ALERT_RULES_PER_ORG} active alert rules`, + ), + ), + ), + Match.tag("Written", (outcome) => + Effect.succeed({ + normalized, + ruleId, + timestamp, + txid: readTxid(outcome.writeRows), + }), + ), + Match.exhaustive, + ) }) const upsertRuleRow = Effect.fn("AlertsService.upsertRuleRow")(function* ( diff --git a/apps/api/src/services/alerts/AlertsService.test.ts b/apps/api/src/services/alerts/AlertsService.test.ts index aab7a4cd0..6ad8ba407 100644 --- a/apps/api/src/services/alerts/AlertsService.test.ts +++ b/apps/api/src/services/alerts/AlertsService.test.ts @@ -3903,3 +3903,139 @@ describe("AlertsService.previewRule", () => { }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub(breachedGroups), { fetch: okFetch }))) }) }) + +describe("AlertsService delivery lease freshness", () => { + it.effect("claims later rows with a fresh timestamp, not the batch head's", () => { + const fixedTime = 1_710_000_400_000 + const testDb = createTestDb(trackedDbs) + // Virtual scheduler clock: the first delivery "takes" longer than the + // lease TTL, so a claim dated from the batch-head timestamp would be + // born expired and an overlapping tick could re-send the event. + let virtualNow = fixedTime + let call = 0 + let laterRowLeaseExpiry: number | null = null + const fetchImpl = (async () => { + call += 1 + if (call === 1) { + virtualNow += 31_000 + } else { + const row = await queryFirstRow<{ claimExpiresAt: Date | null }>( + testDb, + `select claim_expires_at as "claimExpiresAt" + from alert_delivery_events where delivery_key = 'lease-2'`, + [], + ) + laterRowLeaseExpiry = row?.claimExpiresAt?.getTime() ?? null + } + return new Response("ok", { status: 200 }) + }) as typeof fetch + + return Effect.gen(function* () { + yield* TestClock.setTime(fixedTime) + const alerts = yield* AlertsService + const orgId = asOrgId("org_lease_fresh") + const userId = asUserId("user_lease_fresh") + const destination = yield* createWebhookDestination(alerts, orgId, userId) + const rule = yield* createErrorRateRule(alerts, orgId, userId, destination.id) + // Break the rule's stored query so the tick's evaluation half queues + // nothing of its own. + yield* Effect.promise(() => + executeSql(testDb, "update alert_rules set query_spec_json = $1::jsonb where id = $2", [ + "{}", + rule.id, + ]), + ) + + for (const n of [1, 2]) { + yield* Effect.promise(() => + insertDeliveryEventRow(testDb, { + id: `00000000-0000-4000-8000-00000000030${n}`, + orgId, + incidentId: null, + ruleId: rule.id, + destinationId: destination.id, + deliveryKey: `lease-${n}`, + eventType: "test", + attemptNumber: 1, + status: "queued", + // Row 1 first: rows are processed in scheduledAt order. + scheduledAt: fixedTime - 10 + n, + payloadJson: JSON.stringify({ + eventType: "test", + incidentId: null, + incidentStatus: "resolved", + dedupeKey: `lease-${n}`, + observed: { value: 0, sampleCount: 0 }, + }), + }), + ) + } + yield* alerts.runSchedulerTick() + + assert.strictEqual(call, 2) + // The second claim must be dated from its own claim time (after the + // 31s "slow" first delivery), so its lease is still alive while the + // delivery is in flight. A batch-head lease would already be expired. + assert.isNotNull(laterRowLeaseExpiry) + assert.isAbove(laterRowLeaseExpiry ?? 0, fixedTime + 31_000) + }).pipe( + Effect.provide( + makeLayer(testDb, makeWarehouseStub({ tracesAggregateRows: emptyWarehouseRows }), { + fetch: fetchImpl, + now: Effect.sync(() => virtualNow), + }), + ), + ) + }) +}) + +describe("alert incident open uniqueness", () => { + const insertOpenIncident = (testDb: TestDb, id: string, status = "open") => + executeSql( + testDb, + ` + insert into alert_incidents ( + id, org_id, rule_id, incident_key, rule_name, group_key, signal_type, + severity, status, comparator, threshold, first_triggered_at, + last_triggered_at, dedupe_key, created_at, updated_at + ) values ( + $1, 'org_unique', 'rule_unique', $1, 'Rule', 'checkout', 'error_rate', + 'critical', $2, 'gt', 5, now(), now(), 'org_unique:rule_unique:checkout', + now(), now() + ) + `, + [id, status], + ) + + it.effect("rejects a second open incident for the same (rule, group)", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + yield* Effect.promise(() => insertOpenIncident(testDb, "inc-open-1")) + // The partial unique index is the backstop for an expired scheduler + // claim: two overlapping ticks can both decide to open, but only one + // row can exist. + const second = yield* Effect.promise(() => + insertOpenIncident(testDb, "inc-open-2").then( + () => "inserted" as const, + () => "conflict" as const, + ), + ) + assert.strictEqual(second, "conflict") + + // Resolved history does not participate: after resolving the first, + // a new open incident for the same key is legal again. + yield* Effect.promise(() => + executeSql(testDb, "update alert_incidents set status = 'resolved' where id = $1", [ + "inc-open-1", + ]), + ) + const third = yield* Effect.promise(() => + insertOpenIncident(testDb, "inc-open-3").then( + () => "inserted" as const, + () => "conflict" as const, + ), + ) + assert.strictEqual(third, "inserted") + }) + }) +}) diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 729140d66..49fccdfb4 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -89,7 +89,7 @@ import { EmailService } from "@/platform/EmailService" import { Env } from "@/platform/Env" import { OrgClickHouseSettingsService } from "@/services/org/OrgClickHouseSettingsService" import { makeDbExecute } from "@/platform/db-execute" -import { dateToMs, msToDate } from "@/platform/time" +import { dateToMs, msToDate, msToSqlTimestamp } from "@/platform/time" import { makePersistenceError } from "./alert-persistence" import { QueryEngineService } from "@/services/warehouse/QueryEngineService" import type { GroupedAlertObservation } from "@maple/query-engine/runtime" @@ -485,6 +485,7 @@ export class AlertsService extends Context.Service= ${DESTINATION_DISABLE_AFTER_FAILURES}` const counted = yield* dbExecute((db) => db .update(alertDestinations) @@ -1445,6 +1449,12 @@ export class AlertsService extends Context.Service - db - .update(alertDestinations) - .set({ - enabled: false, - disabledAt: msToDate(currentTime), - disabledReason: reason, - updatedAt: msToDate(currentTime), - }) - .where(eq(alertDestinations.id, row.destinationId)), - ) + // None: the `enabled = true` predicate matched no row (already + // disabled elsewhere) or the update left the destination enabled. + const disabled = Option.filter(Arr.head(counted), (row) => !row.enabled) + if (Option.isNone(disabled)) return + const streak = disabled.value.consecutiveFailures // The in-product signal is the setup audit: a disabled destination // makes every rule that selects it fail CFG-ALERT-03 ("will evaluate @@ -1552,7 +1555,13 @@ export class AlertsService extends Context.Service @@ -1673,8 +1682,8 @@ export class AlertsService extends Context.Service db.insert(alertIncidents).values(incident)) + // `alert_incidents_open_group_idx` allows one open incident per + // (org, rule, group). The scheduler claim already serializes rules + // in the common case; this is the backstop for an expired claim — + // a chunk that outran SCHEDULER_LOCK_TTL_MS being re-claimed by the + // next tick, both working from tick-head prefetch that saw no open + // incident. The loser lands here and must not notify. + const inserted = yield* dbExecute((db) => + db.insert(alertIncidents).values(incident).onConflictDoNothing().returning({ + id: alertIncidents.id, + }), + ) + if (inserted.length === 0) { + yield* Effect.logWarning( + "Skipped duplicate incident open: another worker won the race", + ).pipe(Effect.annotateLogs({ ruleId: row.id, groupKey })) + return { + transition: "none" as const, + incidentId: carriedIncidentId, + openedIncidentId: null, + consecutiveBreaches, + consecutiveHealthy, + } + } if (flapSuppressedAt != null) { yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe( Effect.annotateLogs({ diff --git a/apps/api/src/services/alerts/AnomalyDetectionService.test.ts b/apps/api/src/services/alerts/AnomalyDetectionService.test.ts index 2fe52a8ad..51e3b8298 100644 --- a/apps/api/src/services/alerts/AnomalyDetectionService.test.ts +++ b/apps/api/src/services/alerts/AnomalyDetectionService.test.ts @@ -95,7 +95,10 @@ const seedIncident = ( [ asIncidentId(`00000000-0000-4000-8000-0000000000${String(incident.n).padStart(2, "0")}`), incident.orgId, - `${incident.signalType}:${incident.serviceName}:${incident.deploymentEnv}`, + // Suffixed with n: `anomaly_incidents_open_detector_idx` allows one OPEN + // incident per detector, so same-group rows model distinct detectors + // (the error-spike shape, where the fingerprint is part of the key). + `${incident.signalType}:${incident.serviceName}:${incident.deploymentEnv}:${incident.n}`, incident.signalType, incident.serviceName, incident.deploymentEnv, diff --git a/apps/api/src/services/alerts/AnomalyDetectionService.ts b/apps/api/src/services/alerts/AnomalyDetectionService.ts index c867f9d58..f06979335 100644 --- a/apps/api/src/services/alerts/AnomalyDetectionService.ts +++ b/apps/api/src/services/alerts/AnomalyDetectionService.ts @@ -63,6 +63,7 @@ import { Env } from "@/platform/Env" import { dateToMs, msToDate } from "@/platform/time" import { WarehouseQueryService } from "@/services/warehouse/WarehouseQueryService" import { + capBusiestLogSeries, ERROR_SPIKE_MIN_COUNT, evaluateErrorSpike, evaluateGoldenSignals, @@ -1024,7 +1025,7 @@ const make = Effect.gen(function* () { baseline: baseline.get(key) ?? [], }) } - return series.slice(0, MAX_SERIES_PER_ORG) + return capBusiestLogSeries(series, MAX_SERIES_PER_ORG) }) const fetchErrorSpikes = Effect.fn("AnomalyDetectionService.fetchErrorSpikes")(function* ( @@ -1647,7 +1648,22 @@ const make = Effect.gen(function* () { createdAt: new Date(nowMs), updatedAt: new Date(nowMs), } - yield* dbExecute((db) => db.insert(anomalyIncidents).values(insertValues)) + // `anomaly_incidents_open_detector_idx` allows one open incident + // per detector. The org claim is a bare TTL CAS, so a tick that + // outruns ORG_LOCK_TTL_MS can overlap the next one and both can + // reach here for the same detector; the loser must not create a + // second incident or enqueue a second triage. + const insertedIncident = yield* dbExecute((db) => + db.insert(anomalyIncidents).values(insertValues).onConflictDoNothing().returning({ + id: anomalyIncidents.id, + }), + ) + if (insertedIncident.length === 0) { + yield* Effect.logWarning( + "Skipped duplicate anomaly incident open: another tick won the race", + ).pipe(Effect.annotateLogs({ orgId, detectorKey: evaluation.detectorKey })) + return + } const runtime: IncidentRuntime = { row: { ...insertValues, resolvedAt: null, resolveReason: null }, entries, diff --git a/apps/api/src/services/alerts/NotificationDispatcher.test.ts b/apps/api/src/services/alerts/NotificationDispatcher.test.ts new file mode 100644 index 000000000..3e1815fec --- /dev/null +++ b/apps/api/src/services/alerts/NotificationDispatcher.test.ts @@ -0,0 +1,104 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { AlertDestinationId, OrgId } from "@maple/domain/http" +import { Database, DatabaseError } from "@/platform/DatabaseLive" +import { EmailService } from "@/platform/EmailService" +import { Env } from "@/platform/Env" +import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { NotificationDispatcher, type NotificationRequest } from "./NotificationDispatcher" + +const createdDbs: TestDb[] = [] + +afterEach(() => cleanupTestDbs(createdDbs)) + +const testConfig = () => + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3476", + MCP_PORT: "3477", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 1).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + INTERNAL_SERVICE_TOKEN: "test-internal-token", + }), + ) + +const emailStub: (typeof EmailService)["Service"] = { + isConfigured: true, + send: () => Effect.void, +} + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asDestinationId = Schema.decodeUnknownSync(AlertDestinationId) +const ORG = asOrgId("org_dispatcher_test") +const DESTINATION_ID = asDestinationId("00000000-0000-4000-8000-000000000001") + +const request: NotificationRequest = { + deliveryKey: "org_dispatcher_test:dest:delivery", + ruleId: "rule_1", + ruleName: "Checkout error rate", + groupKey: null, + signalType: "error_rate", + severity: "critical", + comparator: "gt", + threshold: 0.05, + eventType: "trigger", + incidentId: null, + incidentStatus: "open", + dedupeKey: "org_dispatcher_test:rule_1", + windowMinutes: 5, + value: 0.08, + sampleCount: 1200, + linkUrl: "https://web.localhost/alerts", +} + +describe("NotificationDispatcher.dispatch", () => { + it.effect("reports a destination-lookup failure as failed, not missing", () => { + // A transient database error must keep the consumers' retry machinery in + // play — "missing" is terminal to the escalation outbox and error + // notification queue, which would silently drop the notification. + const failingDb = Layer.succeed(Database, { + execute: () => Effect.fail(new DatabaseError({ message: "connection reset", cause: null })), + }) + const layer = NotificationDispatcher.layer.pipe( + Layer.provide(Layer.succeed(EmailService, emailStub)), + Layer.provideMerge(failingDb), + Layer.provideMerge(Env.layer), + Layer.provide(testConfig()), + ) + return Effect.gen(function* () { + const dispatcher = yield* NotificationDispatcher + const result = yield* dispatcher.dispatch(ORG, [DESTINATION_ID], request) + + assert.strictEqual(result.delivered, 0) + assert.strictEqual(result.failed, 1) + assert.strictEqual(result.destinations[0]?.status, "failed") + assert.strictEqual(result.destinations[0]?.error, "destination_lookup_failed") + }).pipe(Effect.provide(layer)) + }) + + it.effect("still reports a genuinely absent destination as missing", () => { + const testDb = createTestDb(createdDbs) + const layer = NotificationDispatcher.layer.pipe( + Layer.provide(Layer.succeed(EmailService, emailStub)), + Layer.provideMerge(testDb.layer), + Layer.provideMerge(Env.layer), + Layer.provide(testConfig()), + ) + return Effect.gen(function* () { + const dispatcher = yield* NotificationDispatcher + const result = yield* dispatcher.dispatch(ORG, [DESTINATION_ID], request) + + assert.strictEqual(result.delivered, 0) + // Missing is not a delivery failure: the row does not exist, so there + // is nothing a retry could reach. + assert.strictEqual(result.failed, 0) + assert.strictEqual(result.destinations[0]?.status, "missing") + assert.strictEqual(result.destinations[0]?.error, "destination_missing") + }).pipe(Effect.provide(layer)) + }) +}) diff --git a/apps/api/src/services/alerts/NotificationDispatcher.ts b/apps/api/src/services/alerts/NotificationDispatcher.ts index bc7ae64e9..f6448f413 100644 --- a/apps/api/src/services/alerts/NotificationDispatcher.ts +++ b/apps/api/src/services/alerts/NotificationDispatcher.ts @@ -10,7 +10,7 @@ import { type OrgId, } from "@maple/domain/http" import { and, eq, inArray } from "drizzle-orm" -import { Clock, Context, Effect, Layer, Redacted, Schema } from "effect" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" import { buildAlertChatUrl } from "./AlertDeliveryDispatch" import { dispatchDelivery as dispatchDeliveryImpl } from "./delivery/dispatch" import type { DispatchContext } from "./delivery/context" @@ -232,7 +232,7 @@ const make: Effect.Effect< ) { if (destinationIds.length === 0) return { delivered: 0, failed: 0, destinations: [] } - const rows = yield* database + const rowsOption = yield* database .execute((db) => db .select() @@ -250,12 +250,30 @@ const make: Effect.Effect< Effect.annotateLogs({ orgId, message: error.message }), ), ), - Effect.catchTag("@maple/api/lib/DatabaseError", () => - Effect.succeed>([]), - ), + Effect.asSome, + // A failed lookup must not masquerade as "these destinations do + // not exist": "missing" is terminal to every consumer (escalation + // outbox, error policies), while "failed" keeps their retry + // machinery in play for what is a transient database error. + Effect.catchTag("@maple/api/lib/DatabaseError", () => Effect.succeedNone), ) - const rowsById = new Map(rows.map((row) => [row.id, row])) + if (Option.isNone(rowsOption)) { + return { + delivered: 0, + failed: destinationIds.length, + destinations: destinationIds.map( + (destinationId): NotificationDestinationResult => ({ + destinationId, + destinationName: null, + status: "failed", + error: "destination_lookup_failed", + }), + ), + } + } + + const rowsById = new Map(rowsOption.value.map((row) => [row.id, row])) const results = yield* Effect.forEach( destinationIds, (destinationId) => { diff --git a/apps/api/src/services/alerts/anomaly/detection.test.ts b/apps/api/src/services/alerts/anomaly/detection.test.ts index a16856a61..2d71071b8 100644 --- a/apps/api/src/services/alerts/anomaly/detection.test.ts +++ b/apps/api/src/services/alerts/anomaly/detection.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest" import { + capBusiestLogSeries, evaluateErrorSpike, evaluateGoldenSignals, evaluateLogVolume, @@ -323,3 +324,31 @@ describe("evaluateErrorSpike", () => { expect(e.status).toBe("skipped") }) }) + +describe("capBusiestLogSeries", () => { + const series = (serviceName: string, errorLogCount: number, baselineHours: number) => ({ + serviceName, + deploymentEnv: "production", + current: { errorLogCount }, + baseline: Array.from({ length: baselineHours }, () => ({ errorLogCount: 5 })), + }) + + it("keeps the busiest series regardless of input order", () => { + // ClickHouse GROUP BY order is arbitrary; an unsorted slice could drop + // the busy (or already-anomalous) series and let the no-data sweep + // falsely resolve their open incidents. + const quiet = Array.from({ length: 5 }, (_, i) => series(`quiet-${i}`, 1, 0)) + const busy = series("busy", 5000, 21) + const capped = capBusiestLogSeries([...quiet, busy], 3) + expect(capped[0]?.serviceName).toBe("busy") + expect(capped).toHaveLength(3) + }) + + it("ranks a series with baseline history but no current traffic as busy", () => { + // A series that went dark is exactly the one the detector must evaluate. + const dark = series("dark", 0, 21) + const chatty = series("chatty", 2, 0) + const capped = capBusiestLogSeries([chatty, dark], 1) + expect(capped[0]?.serviceName).toBe("dark") + }) +}) diff --git a/apps/api/src/services/alerts/anomaly/detection.ts b/apps/api/src/services/alerts/anomaly/detection.ts index a2b137504..32f4ab6e7 100644 --- a/apps/api/src/services/alerts/anomaly/detection.ts +++ b/apps/api/src/services/alerts/anomaly/detection.ts @@ -95,6 +95,25 @@ export interface LogVolumeSeries { readonly baseline: ReadonlyArray<{ errorLogCount: number }> } +/** + * Bound per-org work to the busiest log series — the same rule the golden + * path applies inline. The input arrives in ClickHouse GROUP BY order, which + * is arbitrary: slicing it unsorted would drop busy (or already-anomalous) + * series nondeterministically, and a dropped open series never refreshes + * `lastTriggeredAt`, so the no-data sweep would falsely resolve it. + */ +export const capBusiestLogSeries = ( + series: ReadonlyArray, + max: number, +): ReadonlyArray => + [...series] + .sort( + (a, b) => + Math.max(b.current.errorLogCount, b.baseline.length) - + Math.max(a.current.errorLogCount, a.baseline.length), + ) + .slice(0, max) + export interface ErrorSpikeObservation { readonly fingerprintHash: string readonly serviceName: string diff --git a/apps/api/src/services/alerts/delivery/runTransport.ts b/apps/api/src/services/alerts/delivery/runTransport.ts index 04eacbee7..c6eef83dc 100644 --- a/apps/api/src/services/alerts/delivery/runTransport.ts +++ b/apps/api/src/services/alerts/delivery/runTransport.ts @@ -123,19 +123,25 @@ const sendHttp = Effect.fn("AlertDelivery.http", { kind: "client" })(function* ( // which calls it as a bare local. const { fetchFn } = runtime + // The signal is what makes the timeout below real: `timeoutOrElse` interrupts + // this Effect, and without wiring the interruption to `RequestInit.signal` + // the POST would keep running and could still deliver after we reported a + // retryable timeout — a duplicate page once the retry lands. const response = yield* Effect.tryPromise({ - try: () => + try: (signal) => spec.guarded ? safeFetch(spec.url, { method: "POST", headers: { ...spec.headers }, body: spec.body, + signal, fetchFn, }) : fetchFn(spec.url, { method: "POST", headers: { ...spec.headers }, body: spec.body, + signal, }), catch: (error) => makeDeliveryError( diff --git a/apps/api/src/services/alerts/delivery/transports/slack.ts b/apps/api/src/services/alerts/delivery/transports/slack.ts index dc7748950..29fd5b92d 100644 --- a/apps/api/src/services/alerts/delivery/transports/slack.ts +++ b/apps/api/src/services/alerts/delivery/transports/slack.ts @@ -1,6 +1,8 @@ // BOUNDARY: This module intentionally carries opaque values; callers decode them before domain use. import { + AlertDeliveryAuthError, AlertDeliveryError, + AlertDeliveryRejectedError, AlertDeliveryTargetMissingError, type AlertDeliveryFailure, type OrgId, @@ -39,6 +41,34 @@ const slackError = (message: string, providerErrorCode?: string) => ...(!(providerErrorCode === undefined) ? { providerErrorCode } : undefined), }) +/** + * Slack reports every logical failure as HTTP 200 + an error code, so the + * status classifier in `runTransport` never sees them and the code has to be + * mapped here. Anything not listed stays retryable (`AlertDeliveryError`): + * mis-classifying a transient code as terminal costs a destination its + * enablement, while the reverse only costs a few wasted retries. + */ +const SLACK_AUTH_ERRORS = new Set([ + "invalid_auth", + "not_authed", + "token_revoked", + "token_expired", + "account_inactive", + "no_permission", + "missing_scope", + "ekm_access_denied", +]) +const SLACK_TARGET_MISSING_ERRORS = new Set(["not_in_channel", "channel_not_found", "is_archived"]) +const SLACK_REJECTED_ERRORS = new Set([ + "invalid_blocks", + "invalid_blocks_format", + "invalid_arguments", + "msg_too_long", + "too_many_attachments", + "restricted_action", + "cannot_dm_bot", +]) + /** * The bot token is not in the destination's secret config — it is resolved per * org from the `slack_workspaces` row — which is why this is the one transport @@ -98,11 +128,10 @@ export const makeSlackTransport = (deps: SlackTransportDeps): HttpTransport { expect(v.reason).toBe("no_baseline") }) }) + +describe("probeLiveness environment scoping", () => { + // The probe must query the alert's own environments: without the scope, + // staging traffic for the same service satisfies liveness while production + // is dark — exactly the outage the probe exists to veto. + it("vetoes when the rule's environment went dark even though the service is loud elsewhere", async () => { + const tenant = systemTenant(Schema.decodeUnknownSync(OrgId)("org_liveness_env")) + const windowStartMs = Date.parse("2026-06-02T00:10:00.000Z") + const windowEndMs = Date.parse("2026-06-02T00:15:00.000Z") + const baselineStartMs = Date.parse("2026-06-01T23:55:00.000Z") + const baselineEndMs = Date.parse("2026-06-02T00:00:00.000Z") + const observedSql: string[] = [] + + const row = (spanCount: number) => ({ + minutesWithData: spanCount > 0 ? 5 : 0, + spanCount, + estimatedSpanCount: spanCount, + errorCount: 0, + estimatedErrorCount: 0, + lastSeen: "2026-06-02 00:14:00", + }) + + const warehouse: LivenessWarehouse = { + warmRoute: () => Effect.void, + compiledQuery: () => Effect.die("org pulse must not run for a service-scoped probe"), + compiledQueryFirst: (_tenant, compiled) => + Effect.gen(function* () { + const query = compiledQueryOf(compiled) + observedSql.push(query.sql) + const isBaseline = query.sql.includes(formatWarehouseDateTime(baselineStartMs)) + const productionScoped = query.sql.includes("DeploymentEnv = 'production'") + // Production went dark in the verification window; staging keeps + // the UNSCOPED totals healthy in both windows. + const spanCount = productionScoped ? (isBaseline ? 1000 : 0) : isBaseline ? 2000 : 1800 + return yield* query.decodeFirstRow([row(spanCount)]).pipe(Effect.orDie) + }), + } + + const verdict = await Effect.runPromise( + probeLiveness({ + warehouse, + tenant, + serviceNames: ["checkout"], + environments: ["production"], + windowStartMs, + windowEndMs, + baselineStartMs, + baselineEndMs, + }), + ) + + expect(observedSql.some((sql) => sql.includes("DeploymentEnv = 'production'"))).toBe(true) + expect(verdict.dataFlowing).toBe(false) + expect(verdict.reason).toBe("no_data") + }) +}) diff --git a/apps/api/src/services/alerts/telemetry-liveness.ts b/apps/api/src/services/alerts/telemetry-liveness.ts index ab48c1652..ff1616538 100644 --- a/apps/api/src/services/alerts/telemetry-liveness.ts +++ b/apps/api/src/services/alerts/telemetry-liveness.ts @@ -87,6 +87,13 @@ export interface LivenessProbeInput { readonly tenant: TenantContext /** Services the subject is scoped to. Empty probes the org as a whole. */ readonly serviceNames: ReadonlyArray + /** + * Deployment environments the alert is scoped to. Empty means unscoped. + * Only honoured on the per-service path: without it, staging traffic for + * the same service satisfies the probe while production is dark — exactly + * the gap the probe exists to veto. + */ + readonly environments: ReadonlyArray /** The quiet window being interpreted as recovery. */ readonly windowStartMs: number readonly windowEndMs: number @@ -107,16 +114,25 @@ const probeServiceWindow = ( warehouse: LivenessWarehouse, tenant: TenantContext, serviceName: string, + deploymentEnv: string | null, startMs: number, endMs: number, ): Effect.Effect => Effect.gen(function* () { - const compiled = CH.compile(CH.serviceLivenessQuery(), { - orgId: tenant.orgId, - serviceName, - startTime: formatWarehouseDateTime(startMs), - endTime: formatWarehouseDateTime(endMs), - }) + const scopedEnv = Option.fromNullOr(deploymentEnv) + const compiled = CH.compile( + CH.serviceLivenessQuery(Option.isSome(scopedEnv) ? { scopeToEnvironment: true } : {}), + { + orgId: tenant.orgId, + serviceName, + ...Option.match(scopedEnv, { + onNone: () => ({}), + onSome: (deploymentEnv) => ({ deploymentEnv }), + }), + startTime: formatWarehouseDateTime(startMs), + endTime: formatWarehouseDateTime(endMs), + }, + ) const row = yield* warehouse.compiledQueryFirst(tenant, compiled, { profile: "list", context: "telemetryLiveness", @@ -175,15 +191,33 @@ export const probeLiveness: (input: LivenessProbeInput) => Effect.Effect => + serviceNames.flatMap( + ( + serviceName, + ): ReadonlyArray<{ + readonly serviceName: string + readonly deploymentEnv: string | null + }> => + input.environments.length === 0 + ? [{ serviceName, deploymentEnv: null }] + : input.environments.map((deploymentEnv) => ({ + serviceName, + deploymentEnv, + })), + ), + ({ serviceName, deploymentEnv }): Effect.Effect => Effect.all( [ probeServiceWindow( warehouse, tenant, serviceName, + deploymentEnv, windowStartMs, windowEndMs, ), @@ -191,6 +225,7 @@ export const probeLiveness: (input: LivenessProbeInput) => Effect.Effect cleanupTestDbs(trackedDbs)) + +const asOrgId = Schema.decodeUnknownSync(OrgId) +const asUserId = Schema.decodeUnknownSync(UserId) +const ORG = asOrgId("org_oauth_helpers") +const USER = asUserId("user_oauth_helpers") + +const baseConfig = { + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 7).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + MAPLE_INGEST_PUBLIC_URL: "https://ingest.example.com", +} + +const configLive = ConfigProvider.layer(ConfigProvider.fromUnknown(baseConfig)) + +const TOKEN_URL = "https://provider.example.com/oauth/token" + +const tokenConfig: OAuthTokenEndpointConfig = { + tokenUrl: TOKEN_URL, + clientId: "test-client-id", + clientSecret: Redacted.make("test-client-secret"), +} + +const jsonResponse = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + +/** Serve the token endpoint from a per-test handler; everything else 404s. */ +const makeFetch = + (handler: (body: URLSearchParams) => Response): typeof globalThis.fetch => + async (input, init) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url + if (url.startsWith(TOKEN_URL)) { + const text = await new Response(init?.body ?? "").text() + return handler(new URLSearchParams(text)) + } + return jsonResponse({ error: "not_found" }, 404) + } + +const fetchLayer = (handler: (body: URLSearchParams) => Response) => + Layer.succeed(FetchHttpClient.Fetch, makeFetch(handler)) + +/** + * Build the helpers directly (they are a factory, not a service) against the + * test PGlite database, optionally through a wrapped `execute`. + */ +const makeHelpers = (wrapDatabase: (database: DatabaseApi) => DatabaseApi = (database) => database) => + Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + return yield* makeOAuthConnectionHelpers({ + provider: "test-provider", + providerLabel: "TestProvider", + database: wrapDatabase(database), + env, + }) + }) + +const provideBase = (testDb: TestDb, fetch: ReturnType) => + Layer.mergeAll( + testDb.layer, + Env.layer.pipe(Layer.provide(configLive)), + FetchHttpClient.layer.pipe(Layer.provide(fetch)), + ) + +/** Seed an expired connection with a refresh token, so any token read must refresh. */ +const seedExpiredConnection = (helpers: Effect.Success>) => + Effect.gen(function* () { + const accessEnc = yield* helpers.encryptValue("stale-access-token") + const refreshEnc = yield* helpers.encryptValue("stored-refresh-token") + yield* helpers.upsertConnection(ORG, Date.now(), { + externalUserId: "ext-user-1", + connectedByUserId: USER, + accessTokenCiphertext: accessEnc.ciphertext, + accessTokenIv: accessEnc.iv, + accessTokenTag: accessEnc.tag, + refreshTokenCiphertext: refreshEnc.ciphertext, + refreshTokenIv: refreshEnc.iv, + refreshTokenTag: refreshEnc.tag, + // Already expired — getValidConnectionToken must refresh. + expiresAt: new Date(Date.now() - 60_000), + }) + }) + +const revokedAtOf = (testDb: TestDb) => + Effect.promise(() => + queryFirstRow<{ revoked_at: string | null }>( + testDb, + "SELECT revoked_at FROM oauth_connections WHERE org_id = $1", + [ORG], + ), + ).pipe(Effect.map((row) => row?.revoked_at ?? null)) + +describe("refreshAccessToken classification", () => { + it.live("a 400 invalid_grant is a revocation and stamps the connection revoked", () => { + const testDb = createTestDb(trackedDbs) + const fetch = fetchLayer(() => jsonResponse({ error: "invalid_grant" }, 400)) + return Effect.gen(function* () { + const helpers = yield* makeHelpers() + yield* seedExpiredConnection(helpers) + const error = yield* helpers.getValidConnectionToken(tokenConfig, ORG).pipe(Effect.flip) + assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsRevokedError") + assert.isNotNull(yield* revokedAtOf(testDb)) + }).pipe(Effect.provide(provideBase(testDb, fetch))) + }) + + it.live("a 400 invalid_client is an upstream failure and does NOT revoke the connection", () => { + const testDb = createTestDb(trackedDbs) + // A rotated/misconfigured Maple client secret answers this for EVERY + // tenant at once — stamping revoked here would disconnect them all. + const fetch = fetchLayer(() => jsonResponse({ error: "invalid_client" }, 400)) + return Effect.gen(function* () { + const helpers = yield* makeHelpers() + yield* seedExpiredConnection(helpers) + const error = yield* helpers.getValidConnectionToken(tokenConfig, ORG).pipe(Effect.flip) + assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsUpstreamError") + assert.isNull(yield* revokedAtOf(testDb)) + }).pipe(Effect.provide(provideBase(testDb, fetch))) + }) + + it.live("a bodyless 401 is an upstream failure and does NOT revoke the connection", () => { + const testDb = createTestDb(trackedDbs) + const fetch = fetchLayer(() => new Response("Unauthorized", { status: 401 })) + return Effect.gen(function* () { + const helpers = yield* makeHelpers() + yield* seedExpiredConnection(helpers) + const error = yield* helpers.getValidConnectionToken(tokenConfig, ORG).pipe(Effect.flip) + assert.strictEqual(error._tag, "@maple/http/errors/IntegrationsUpstreamError") + assert.isNull(yield* revokedAtOf(testDb)) + }).pipe(Effect.provide(provideBase(testDb, fetch))) + }) +}) + +describe("persistRefreshedTokens", () => { + it.live("retries a transient persistence failure instead of losing the rotated tokens", () => { + const testDb = createTestDb(trackedDbs) + const fetch = fetchLayer(() => + jsonResponse({ + access_token: "fresh-access-token", + refresh_token: "fresh-refresh-token", + expires_in: 3600, + }), + ) + let failuresLeft = 2 + const flaky = (database: DatabaseApi): DatabaseApi => ({ + // Suspended so each retry re-evaluates the failure budget. + execute: (fn) => + Effect.suspend(() => { + if (failuresLeft > 0) { + failuresLeft -= 1 + return Effect.fail(new DatabaseError({ message: "connection reset", cause: "boom" })) + } + return database.execute(fn) + }), + }) + return Effect.gen(function* () { + const helpers = yield* makeHelpers() + yield* seedExpiredConnection(helpers) + const row = yield* helpers.requireConnection(ORG) + + // From here every execute goes through the flaky wrapper: the refresh + // succeeded upstream (the old refresh token is now dead), so the write + // must survive a transient blip. + const flakyHelpers = yield* makeHelpers(flaky) + const refreshed = yield* flakyHelpers.refreshAccessToken(tokenConfig, "stored-refresh-token") + const accessToken = yield* flakyHelpers.persistRefreshedTokens(row, refreshed) + + assert.strictEqual(accessToken, "fresh-access-token") + assert.strictEqual(failuresLeft, 0) + const persisted = yield* helpers.getValidConnectionToken(tokenConfig, ORG) + assert.strictEqual(persisted.accessToken, "fresh-access-token") + }).pipe(Effect.provide(provideBase(testDb, fetch))) + }) +}) diff --git a/apps/api/src/services/auth/oauth/connection-helpers.ts b/apps/api/src/services/auth/oauth/connection-helpers.ts index 82ba1d77c..d575152be 100644 --- a/apps/api/src/services/auth/oauth/connection-helpers.ts +++ b/apps/api/src/services/auth/oauth/connection-helpers.ts @@ -16,7 +16,7 @@ import { } from "@maple/domain/http" import { oauthAuthStates, oauthConnections, type OAuthAuthStateRow, type OAuthConnectionRow } from "@maple/db" import { and, eq, isNull, lt } from "drizzle-orm" -import { Clock, Effect, Redacted, Schema, Semaphore } from "effect" +import { Clock, Effect, Option, Redacted, Schedule, Schema, Semaphore } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" import { decryptAes256Gcm, @@ -63,6 +63,20 @@ export type OAuthTokenResponse = typeof OAuthTokenResponseSchema.Type const decodeTokenResponse = Schema.decodeUnknownEffect(OAuthTokenResponseSchema) +/** + * RFC 6749 §5.2 error body. Token endpoints answer 400/401 for many reasons — + * `invalid_client` (rotated/misconfigured client secret), `invalid_request`, + * `invalid_scope` — and only `invalid_grant` means the grant itself is dead. + * Classifying any other 400/401 as revoked would let one bad Maple client + * secret stamp every tenant connection revoked at once. + */ +const decodeOAuthErrorCode = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Struct({ error: Schema.String })), +) + +const oauthErrorCodeOf = (text: string): Option.Option => + Option.map(decodeOAuthErrorCode(text), (body) => body.error) + export const toUpstreamError = (message: string, status?: number, cause?: unknown) => new IntegrationsUpstreamError({ message, @@ -346,9 +360,11 @@ export const makeOAuthConnectionHelpers = (options: MakeOAuthConnectionHelpersOp ) /** - * Refresh-grant call with the shared classification rule: a 400/401 means - * the grant itself is gone (revoked / rotated away), not a transient - * upstream failure. + * Refresh-grant call with the shared classification rule: only a 400/401 + * whose RFC 6749 error body says `invalid_grant` means the grant itself is + * gone (revoked / rotated away). Every other failure — `invalid_client` + * from a rotated Maple secret, a bodyless 400, 429s, 5xx — is a + * non-mutating upstream failure and must not stamp the connection revoked. */ const refreshAccessToken = Effect.fn("OAuthConnectionHelpers.refreshAccessToken")(function* ( config: OAuthTokenEndpointConfig, @@ -361,10 +377,24 @@ export const makeOAuthConnectionHelpers = (options: MakeOAuthConnectionHelpersOp ...(config.clientSecret ? { client_secret: Redacted.value(config.clientSecret) } : undefined), }) if (status === 400 || status === 401) { + const errorCode = oauthErrorCodeOf(text) + // Only a decoded `invalid_grant` means revoked: a None (bodyless or + // undecodable 400/401) stays a transient upstream failure below. + if (Option.contains(errorCode, "invalid_grant")) { + return yield* Effect.fail( + new IntegrationsRevokedError({ + message: `${providerLabel} connection no longer authorized — reconnect required`, + }), + ) + } return yield* Effect.fail( - new IntegrationsRevokedError({ - message: `${providerLabel} connection no longer authorized — reconnect required`, - }), + toUpstreamError( + `Token refresh failed with ${status}${Option.match(errorCode, { + onNone: () => "", + onSome: (code) => ` (${code})`, + })}`, + status, + ), ) } if (status < 200 || status >= 300) { @@ -398,6 +428,11 @@ export const makeOAuthConnectionHelpers = (options: MakeOAuthConnectionHelpersOp updatedAt: new Date(currentTime), }) .where(eq(oauthConnections.id, row.id)), + ).pipe( + // The provider already rotated the refresh token, so this write holds the + // only usable copy — losing it to a transient Postgres blip turns into a + // permanent disconnect on the next refresh. Retry hard before giving up. + Effect.retry({ times: 3, schedule: Schedule.exponential("100 millis") }), ) return tokenResponse.access_token }) diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts index a4ca5ae30..064531713 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.test.ts @@ -14,9 +14,11 @@ import { errorIssueEvents, errorIssuePullRequests, errorIssueStates, + issueEscalations, } from "@maple/db" +import type { MapleDatabaseTransaction } from "@maple/db/client" import { and, eq } from "drizzle-orm" -import { Database } from "@/platform/DatabaseLive" +import { Database, type DatabaseApi, type DatabaseClient } from "@/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { ErrorActorsService } from "./ErrorActorsService" import { ErrorIssueWorkflowService } from "./ErrorIssueWorkflowService" @@ -46,6 +48,56 @@ const makeLayer = () => { return Layer.mergeAll(workflow, actors).pipe(Layer.provideMerge(database)) } +/** + * The client with one table's inserts sabotaged, inside and outside + * transactions — a stand-in for the connection dying mid-write, which is what + * the workflow's multi-statement operations must survive atomically. + */ +const failInsertOf = (client: T, failTable: unknown): T => + new Proxy(client, { + get(target, property) { + // SAFETY: a Proxy get trap receives a key for its target; indexed access preserves + // the target's own property type while the runtime branch below validates callability. + const value = target[property as keyof T] + if (typeof value !== "function") return value + if (property === "insert") { + return (table: unknown) => { + if (table === failTable) throw new Error("injected insert failure") + return value.call(target, table) + } + } + if (property === "transaction") { + return ( + callback: (tx: MapleDatabaseTransaction) => Promise, + ...rest: ReadonlyArray + ) => + value.call( + target, + (tx: MapleDatabaseTransaction) => callback(failInsertOf(tx, failTable)), + ...rest, + ) + } + return value.bind(target) + }, + }) + +const makeFaultyLayer = (failTable: unknown) => { + const database = createTestDb(createdDbs).layer + const faulty = Layer.effect( + Database, + Effect.gen(function* () { + const real = yield* Database + return { + execute: (fn: (db: DatabaseClient) => Promise) => + real.execute((db) => fn(failInsertOf(db, failTable))), + } satisfies DatabaseApi + }), + ).pipe(Layer.provide(database)) + const actors = ErrorActorsService.layer.pipe(Layer.provide(faulty)) + const workflow = databaseAndActorsOnly.pipe(Layer.provide(Layer.mergeAll(faulty, actors))) + return Layer.mergeAll(workflow, actors).pipe(Layer.provideMerge(faulty)) +} + const seedIssue = (issueId: ErrorIssueId, overrides: Partial = {}) => Effect.gen(function* () { const database = yield* Database @@ -176,6 +228,78 @@ describe("ErrorIssueWorkflowService", () => { }).pipe(Effect.provide(makeLayer())), ) + it.effect("rolls the whole done transition back when the timeline event cannot commit", () => + Effect.gen(function* () { + const workflow = yield* ErrorIssueWorkflowService + const actors = yield* ErrorActorsService + const database = yield* Database + const actor = yield* actors.ensureUserActor(ORG, USER) + const issueId = asIssueId(randomUUID()) + const incidentId = asIncidentId(randomUUID()) + const now = yield* Clock.currentTimeMillis + yield* seedIssue(issueId, { workflowState: "in_review" }) + yield* database.execute((db) => + db.insert(errorIncidents).values({ + id: incidentId, + orgId: ORG, + issueId, + status: "open", + reason: "first_seen", + firstTriggeredAt: new Date(now), + lastTriggeredAt: new Date(now), + createdAt: new Date(now), + updatedAt: new Date(now), + }), + ) + + const current = yield* workflow.requireIssue(ORG, issueId) + const failure = yield* Effect.flip(workflow.applyTransition(ORG, actor.id, current, "done")) + assert.strictEqual(failure._tag, "@maple/http/errors/ErrorPersistenceError") + + // Nothing may commit without the event: a done issue with an open + // incident and no audit trail is unrepairable, because a retry sees the + // target state already stored and returns early. + const after = yield* workflow.requireIssue(ORG, issueId) + assert.strictEqual(after.workflowState, "in_review") + assert.isNull(after.resolvedAt) + const [incident] = yield* database.execute((db) => + db.select().from(errorIncidents).where(eq(errorIncidents.id, incidentId)), + ) + assert.strictEqual(incident?.status, "open") + }).pipe(Effect.provide(makeFaultyLayer(errorIssueEvents))), + ) + + it.effect("rolls the severity change back when the escalation outbox insert fails", () => + Effect.gen(function* () { + const workflow = yield* ErrorIssueWorkflowService + const actors = yield* ErrorActorsService + const database = yield* Database + const actor = yield* actors.ensureUserActor(ORG, USER) + const issueId = asIssueId(randomUUID()) + yield* seedIssue(issueId) + + const failure = yield* Effect.flip( + workflow.setSeverity(ORG, actor.id, issueId, "critical", { source: "manual" }), + ) + assert.strictEqual(failure._tag, "@maple/http/errors/ErrorPersistenceError") + + // The severity must not outlive its escalation row: committed alone, a + // retried setSeverity observes "nothing changed" and returns before + // enqueueing, so the page for this severity is permanently lost. + const [issue] = yield* database.execute((db) => + db.select().from(errorIssues).where(eq(errorIssues.id, issueId)), + ) + assert.isNull(issue?.severity) + const events = yield* database.execute((db) => + db + .select() + .from(errorIssueEvents) + .where(and(eq(errorIssueEvents.orgId, ORG), eq(errorIssueEvents.issueId, issueId))), + ) + assert.deepStrictEqual(events, []) + }).pipe(Effect.provide(makeFaultyLayer(issueEscalations))), + ) + it.effect("hydrates activity rollups: comments, agent notes, and non-abandoned PR links", () => Effect.gen(function* () { const workflow = yield* ErrorIssueWorkflowService diff --git a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts index 875d91b8e..6bd1facbb 100644 --- a/apps/api/src/services/errors/ErrorIssueWorkflowService.ts +++ b/apps/api/src/services/errors/ErrorIssueWorkflowService.ts @@ -152,6 +152,22 @@ export interface ErrorIssueWorkflowServiceApi extends ErrorIssueWorkflowPublicAp readonly timestamp?: number }, ) => Effect.Effect + /** + * The insert `recordEvent` would write, for a caller that must commit the + * event atomically with its own statements in one transaction. + */ + readonly buildEvent: ( + orgId: OrgId, + issueId: ErrorIssueId, + actorId: ActorId | null, + type: ErrorIssueEventType, + timestamp: number, + opts?: { + readonly fromState?: WorkflowState | null + readonly toState?: WorkflowState | null + readonly payload?: StoredJsonRecord + }, + ) => ErrorIssueEventInsert readonly applyTransition: ( orgId: OrgId, actorId: ActorId | null, @@ -392,21 +408,34 @@ const make: Effect.Effect ({ + id: newEventId(), + orgId, + issueId, + actorId, + type, + fromState: opts.fromState ?? null, + toState: opts.toState ?? null, + payloadJson: opts.payload ?? {}, + createdAt: msToDate(timestamp), + }) + const recordEvent: ErrorIssueWorkflowServiceApi["recordEvent"] = Effect.fn( "ErrorsService.recordEvent", )(function* (orgId, issueId, actorId, type, opts = {}) { const timestamp = opts.timestamp ?? (yield* Clock.currentTimeMillis) - const insert: ErrorIssueEventInsert = { - id: newEventId(), - orgId, - issueId, - actorId: actorId ?? null, - type, - fromState: opts.fromState ?? null, - toState: opts.toState ?? null, - payloadJson: opts.payload ?? {}, - createdAt: msToDate(timestamp), - } + const insert = buildEventInsert(orgId, issueId, actorId ?? null, type, timestamp, opts) return yield* dbExecute((db) => db.insert(errorIssueEvents).values(insert)) }) @@ -485,46 +514,48 @@ const make: Effect.Effect - db - .update(errorIssues) - .set(update) - .where(and(eq(errorIssues.orgId, orgId), eq(errorIssues.id, row.id))), - ) - if (toState === "done") { - yield* dbExecute((db) => - db - .update(errorIncidents) - .set({ - status: "resolved", - resolvedAt: msToDate(timestamp), - updatedAt: msToDate(timestamp), - }) - .where( - and( - eq(errorIncidents.orgId, orgId), - eq(errorIncidents.issueId, row.id), - eq(errorIncidents.status, "open"), - ), - ), - ) - yield* dbExecute((db) => - db - .update(errorIssueStates) - .set({ openIncidentId: null, updatedAt: msToDate(timestamp) }) - .where(and(eq(errorIssueStates.orgId, orgId), eq(errorIssueStates.issueId, row.id))), - ) - } - const notePayload: StoredJsonRecord = opts.note ? { ...opts.payload, note: opts.note } : { ...opts.payload } - yield* recordEvent(orgId, row.id, actorId, "state_change", { + const eventInsert = buildEventInsert(orgId, row.id, actorId ?? null, "state_change", timestamp, { fromState, toState, payload: notePayload, - timestamp, }) + // One transaction, because a `done` that committed the issue but lost the + // incident resolution or its timeline event could never be repaired: a + // retry sees `fromState === toState` and returns before reaching them. + yield* dbExecute((db) => + db.transaction(async (tx) => { + await tx + .update(errorIssues) + .set(update) + .where(and(eq(errorIssues.orgId, orgId), eq(errorIssues.id, row.id))) + if (toState === "done") { + await tx + .update(errorIncidents) + .set({ + status: "resolved", + resolvedAt: msToDate(timestamp), + updatedAt: msToDate(timestamp), + }) + .where( + and( + eq(errorIncidents.orgId, orgId), + eq(errorIncidents.issueId, row.id), + eq(errorIncidents.status, "open"), + ), + ) + await tx + .update(errorIssueStates) + .set({ openIncidentId: null, updatedAt: msToDate(timestamp) }) + .where( + and(eq(errorIssueStates.orgId, orgId), eq(errorIssueStates.issueId, row.id)), + ) + } + await tx.insert(errorIssueEvents).values(eventInsert) + }), + ) if (actorId) yield* actors.touchActor(orgId, actorId, timestamp) return yield* requireIssue(orgId, row.id) }) @@ -683,40 +714,33 @@ const make: Effect.Effect - db - .insert(issueEscalations) - .values({ - id: newIssueEscalationId(), - orgId, - issueId, - severity: to, - source, - reason, - runId: null, - investigationId: null, - payloadJson: {}, - deliveryResultsJson: [], - status: "queued", - attempts: 0, - dedupeKey: escalationDedupeKey(orgId, issueId, to), - error: null, - createdAt: msToDate(timestamp), - processedAt: null, - }) - .onConflictDoNothing(), - ) - }) + timestamp: number, + ): Option.Option => + Option.map(Option.fromNullOr(escalationReasonFor(from, to)), (reason) => ({ + id: newIssueEscalationId(), + orgId, + issueId, + severity: to, + source, + reason, + runId: null, + investigationId: null, + payloadJson: {}, + deliveryResultsJson: [], + status: "queued", + attempts: 0, + dedupeKey: escalationDedupeKey(orgId, issueId, to), + error: null, + createdAt: msToDate(timestamp), + processedAt: null, + })) const setSeverity: ErrorIssueWorkflowServiceApi["setSeverity"] = Effect.fn( "ErrorsService.setSeverity", @@ -738,29 +762,43 @@ const make: Effect.Effect + severityEscalationInsert(orgId, issueId, current.severity, next, source, timestamp), + ) + // One transaction: a severity that committed without its escalation row + // could never page anyone — a retry sees the severity already stored and + // returns before reaching the outbox insert. const severityRows = yield* dbExecute((db) => - db - .update(errorIssues) - .set({ - severity, - severitySource: nextSource, - updatedAt: msToDate(timestamp), - }) - .where(and(eq(errorIssues.orgId, orgId), eq(errorIssues.id, issueId))) - .returning(txidColumn), + db.transaction(async (tx) => { + const rows = await tx + .update(errorIssues) + .set({ + severity, + severitySource: nextSource, + updatedAt: msToDate(timestamp), + }) + .where(and(eq(errorIssues.orgId, orgId), eq(errorIssues.id, issueId))) + .returning(txidColumn) + if (Option.isSome(eventInsert)) + await tx.insert(errorIssueEvents).values(eventInsert.value) + if (Option.isSome(escalationInsert)) { + await tx.insert(issueEscalations).values(escalationInsert.value).onConflictDoNothing() + } + return rows + }), ) - if (current.severity !== severity) { - const payload: StoredJsonRecord = opts?.note - ? { from: current.severity, to: severity, source, note: opts.note } - : { from: current.severity, to: severity, source } - yield* recordEvent(orgId, issueId, actorId, "severity_change", { - payload, - timestamp, - }) - } - if (severity !== null) { - yield* enqueueSeverityEscalation(orgId, issueId, current.severity, severity, source) - } yield* actors.touchActor(orgId, actorId, timestamp) const next = yield* requireIssue(orgId, issueId) const doc = yield* hydrateIssue(orgId, next) @@ -827,6 +865,7 @@ const make: Effect.Effect { assert.strictEqual(verificationVerdictAutoCloses("critical"), false) }) }) + +describe("splitVersionRows", () => { + it("splits attributable occurrences against the merge-time baseline", () => { + const split = splitVersionRows( + [ + { serviceVersion: "v1", count: 40 }, + { serviceVersion: "v3", count: 2 }, + ], + ["v1", "v2"], + 1000, + ) + assert.deepStrictEqual(split, { postMerge: 2, staleClients: 40, unattributed: 0 }) + }) + + it("keeps a count of occurrences that report no build", () => { + // The old code discarded these rows entirely, so a service that never + // reports `service.version` looked identical to one that went silent — and + // with a usable pre-merge rate the no-agent fallback then wrote `verified` + // and auto-closed the issue while the error was still firing. + const split = splitVersionRows([{ serviceVersion: "", count: 17 }], ["v1"], 1000) + assert.strictEqual(split.unattributed, 17) + assert.strictEqual(split.postMerge, 0) + }) + + it("treats a truncated version scan as incomplete evidence", () => { + // Any single non-baseline build is decisive, so a scan that dropped rows + // past its cap cannot claim the window was clean. + const rows = Array.from({ length: 3 }, (_, index) => ({ + serviceVersion: `v${index}`, + count: 1, + })) + const split = splitVersionRows(rows, ["v0", "v1", "v2"], 3) + assert.isAbove(split.unattributed, 0) + }) +}) diff --git a/apps/api/src/services/errors/FixVerificationTickService.ts b/apps/api/src/services/errors/FixVerificationTickService.ts index 7ae13e282..10820bbb9 100644 --- a/apps/api/src/services/errors/FixVerificationTickService.ts +++ b/apps/api/src/services/errors/FixVerificationTickService.ts @@ -26,6 +26,45 @@ const decodeRoleNameSync = Schema.decodeUnknownSync(RoleName) */ const MAX_VERIFICATIONS_PER_TICK = 20 +/** + * Distinct versions read per occurrence scan. Membership against the baseline + * is the decision, so a build dropped from the result could be the single + * post-merge build that refutes the fix — a full page is treated as incomplete + * evidence below rather than silently classified from the head. + */ +const VERSION_SCAN_LIMIT = 1000 + +/** + * Split one window's version rows against the merge-time baseline. + * + * Occurrences with no reported build cannot be attributed either way: counting + * one as post-merge would refute every fix from a service that does not report + * `service.version`, and counting it as a stale client would hide a real + * failure. They are tallied as `unattributed`, and any nonzero tally (or a + * truncated scan) must force the verdict toward `inconclusive` — evidence that + * exists but cannot be read is not a clean window. + */ +export const splitVersionRows = ( + rows: ReadonlyArray<{ readonly serviceVersion: string; readonly count: number }>, + baselineVersions: ReadonlyArray, + scanLimit: number, +): { postMerge: number; staleClients: number; unattributed: number } => { + const baseline = new Set(baselineVersions) + let postMerge = 0 + let staleClients = 0 + let unattributed = 0 + for (const entry of rows) { + if (entry.serviceVersion === "") unattributed += entry.count + else if (baseline.has(entry.serviceVersion)) staleClients += entry.count + else postMerge += entry.count + } + // A full page means versions beyond the cap were dropped, and any one of + // them could be decisive. Marked unattributed so the verdict cannot claim + // the window was clean. + if (rows.length >= scanLimit) unattributed += 1 + return { postMerge, staleClients, unattributed } +} + /** * Turn a finished verification investigation into a verdict. * @@ -119,7 +158,7 @@ const make: Effect.Effect< nowMs: number, ) { const mergedAtMs = dateToMs(row.mergedAt) ?? nowMs - const compiled = CH.compile(CH.errorIssueVersionsSinceQuery(), { + const compiled = CH.compile(CH.errorIssueVersionsSinceQuery({ limit: VERSION_SCAN_LIMIT }), { orgId: row.orgId, fingerprintHash, startTime: formatWarehouseDateTime(mergedAtMs), @@ -128,20 +167,7 @@ const make: Effect.Effect< const rows = yield* warehouse.compiledQuery(systemTenant(row.orgId), compiled, { context: "errorIssueVersionsSince", }) - const baseline = new Set(row.baselineVersionsJson) - let postMerge = 0 - let staleClients = 0 - for (const entry of rows) { - // An occurrence with no reported build cannot be attributed either way. - // Counting it as post-merge would refute every fix from a service that - // does not report `service.version`; counting it as a stale client would - // hide a real failure. It is counted as neither, and its absence is what - // makes the verdict "inconclusive" rather than confidently wrong. - if (entry.serviceVersion === "") continue - if (baseline.has(entry.serviceVersion)) staleClients += entry.count - else postMerge += entry.count - } - return { postMerge, staleClients } + return splitVersionRows(rows, row.baselineVersionsJson, VERSION_SCAN_LIMIT) }) const verdictFromRun = verdictFromInvestigationStatus @@ -233,7 +259,13 @@ const make: Effect.Effect< error: summarizeCause(cause), }), // Leave the row waiting; the warehouse being down is not a verdict. - Effect.as(Option.none<{ postMerge: number; staleClients: number }>()), + Effect.as( + Option.none<{ + postMerge: number + staleClients: number + unattributed: number + }>(), + ), ), ), ) @@ -255,6 +287,23 @@ const make: Effect.Effect< continue } + // Occurrences that could not be attributed to a build make the window + // unreadable: proceeding as though they did not happen is how an error + // still firing from a version-less service gets `verified` and + // auto-closed. Inconclusive, not a refutation — those occurrences may + // equally be old clients — and `applyVerdict` re-arms one longer window + // before handing the issue back to a human. + if (split.value.unattributed > 0) { + const applied = yield* applyVerdictGuarded( + row, + "inconclusive", + "Occurrences since the merge carried no service.version (or the version scan was truncated), so they cannot be attributed to a pre- or post-merge build.", + ) + if (applied) verdictsApplied += 1 + else failedRows += 1 + continue + } + // Nothing observed at all, from any build, and the window has run its // course. That is only meaningful if the issue had enough traffic for // silence to mean something — which is exactly what the window length diff --git a/apps/api/src/services/errors/IssueFixVerificationService.test.ts b/apps/api/src/services/errors/IssueFixVerificationService.test.ts index 6383d30c5..278d1f629 100644 --- a/apps/api/src/services/errors/IssueFixVerificationService.test.ts +++ b/apps/api/src/services/errors/IssueFixVerificationService.test.ts @@ -4,8 +4,9 @@ import { Cause, ConfigProvider, Effect, Exit, Layer, Option, Schema } from "effe import { OrgId, type PullRequestSummary, type WorkflowState } from "@maple/domain/http" import { ErrorIssueId } from "@maple/domain/primitives" import { errorIssues, errorIssueEvents, errorIssueVerifications } from "@maple/db" +import type { MapleDatabaseTransaction } from "@maple/db/client" import { eq } from "drizzle-orm" -import { Database } from "@/platform/DatabaseLive" +import { Database, type DatabaseApi, type DatabaseClient } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { ErrorActorsService } from "./ErrorActorsService" @@ -55,12 +56,15 @@ const testConfig = () => * integration does — so every test written before hydration existed keeps * exercising the unenriched path. */ -const makeLayer = (lookup?: { - readonly pullRequest?: () => PullRequestSummary | undefined - readonly repositories?: ReadonlyArray -}) => { - const testDb = createTestDb(createdDbs) - const databaseLive = testDb.layer +const makeLayer = ( + lookup?: { + readonly pullRequest?: () => PullRequestSummary | undefined + readonly repositories?: ReadonlyArray + }, + // Overridable so a test can build two service instances over one database — + // one healthy, one with a sabotaged client — and observe the same rows. + databaseLive: Layer.Layer = createTestDb(createdDbs).layer, +) => { const envLive = Env.layer.pipe(Layer.provide(testConfig())) const actorsLive = ErrorActorsService.layer.pipe(Layer.provide(databaseLive)) const workflowLive = ErrorIssueWorkflowService.layer.pipe( @@ -355,8 +359,7 @@ describe("linkPullRequest hydration", () => { Effect.gen(function* () { const mergedAtMs = Date.now() - HOUR const layer = makeLayer({ - pullRequest: () => - summary({ state: "merged", mergedAtMs, mergeCommitSha: "a".repeat(40) }), + pullRequest: () => summary({ state: "merged", mergedAtMs, mergeCommitSha: "a".repeat(40) }), }) yield* Effect.gen(function* () { const service = yield* IssueFixVerificationService @@ -509,7 +512,6 @@ describe("unlinkPullRequest", () => { }), ) - it.effect("removes the link and abandons any verification riding on it", () => Effect.gen(function* () { const layer = makeLayer() @@ -639,6 +641,29 @@ describe("onPullRequestEvent — merge", () => { }), ) + // Queue deliveries are unordered: a pre-merge event (opened/edited/synchronize) + // can land after the merged one. A GitHub merge is irreversible, so it must + // never regress the link's state or null the merge metadata the verification + // window was opened from. + it.effect("a stale non-merged event cannot regress a merged link", () => + Effect.gen(function* () { + const layer = makeLayer() + yield* Effect.gen(function* () { + const service = yield* IssueFixVerificationService + const issueId = yield* seedIssue() + yield* service.linkPullRequest(ORG, null, issueId, PR_URL, "agent") + yield* service.onPullRequestEvent(mergeEvent()) + // The out-of-order "edited" from before the merge arrives late. + yield* service.onPullRequestEvent( + mergeEvent({ action: "edited", merged: false, mergedAtMs: null, mergeCommitSha: null }), + ) + const links = yield* service.listPullRequests(ORG, issueId) + expect(links.pullRequests[0]?.state).toBe("merged") + expect(links.pullRequests[0]?.mergeCommitSha).toBe("abc123") + }).pipe(Effect.provide(layer)) + }), + ) + it.effect("verifies a merge even on an issue somebody already closed", () => Effect.gen(function* () { const layer = makeLayer() @@ -882,3 +907,131 @@ describe("dueVerifications", () => { }), ) }) + +/** + * The client with one table's inserts sabotaged, inside and outside + * transactions — a stand-in for the connection dying mid-write, which + * `applyVerdict` must survive without settling a row it did not fully record. + */ +const failInsertOf = (client: T, failTable: unknown): T => + new Proxy(client, { + get(target, property) { + // SAFETY: a Proxy get trap receives a key for its target; indexed access preserves + // the target's own property type while the runtime branch below validates callability. + const value = target[property as keyof T] + if (typeof value !== "function") return value + if (property === "insert") { + return (table: unknown) => { + if (table === failTable) throw new Error("injected insert failure") + return value.call(target, table) + } + } + if (property === "transaction") { + return ( + callback: (tx: MapleDatabaseTransaction) => Promise, + ...rest: ReadonlyArray + ) => + value.call( + target, + (tx: MapleDatabaseTransaction) => callback(failInsertOf(tx, failTable)), + ...rest, + ) + } + return value.bind(target) + }, + }) + +const failingEventInsertLayer = (base: Layer.Layer) => + Layer.effect( + Database, + Effect.gen(function* () { + const real = yield* Database + return { + execute: (fn: (db: DatabaseClient) => Promise) => + real.execute((db) => fn(failInsertOf(db, errorIssueEvents))), + } satisfies DatabaseApi + }), + ).pipe(Layer.provide(base)) + +describe("applyVerdict — race and failure discipline", () => { + it.effect("a stale verified verdict cannot overwrite a decisive refutation", () => + Effect.gen(function* () { + const layer = makeLayer() + yield* Effect.gen(function* () { + const service = yield* IssueFixVerificationService + const issueId = yield* seedIssue({ severity: "low", seenVersions: ["v1", "v2"] }) + yield* service.linkPullRequest(ORG, null, issueId, PR_URL, "agent") + yield* service.onPullRequestEvent(mergeEvent()) + + // The tick read the row while it was still waiting… + const stale = yield* readVerification(issueId) + assert.isDefined(stale) + // …and the error tick refuted it before the tick's verdict landed. + expect(yield* service.refuteOnPostMergeOccurrence(ORG, issueId, ["v3"], Date.now())).toBe( + true, + ) + + yield* service.applyVerdict(stale, "verified", "computed from a stale window", Date.now()) + + // The refutation must stand: no overwrite, no auto-close. + expect((yield* readVerification(issueId))?.status).toBe("not_fixed") + expect((yield* readIssueState(issueId))?.workflowState).toBe("in_progress") + }).pipe(Effect.provide(layer)) + }), + ) + + it.effect("markRunning cannot resurrect a refuted verification", () => + Effect.gen(function* () { + const layer = makeLayer() + yield* Effect.gen(function* () { + const service = yield* IssueFixVerificationService + const issueId = yield* seedIssue({ seenVersions: ["v1", "v2"] }) + yield* service.linkPullRequest(ORG, null, issueId, PR_URL, "agent") + yield* service.onPullRequestEvent(mergeEvent()) + + const stale = yield* readVerification(issueId) + assert.isDefined(stale) + expect(yield* service.refuteOnPostMergeOccurrence(ORG, issueId, ["v3"], Date.now())).toBe( + true, + ) + + yield* service.markRunning(stale, null, Date.now()) + expect((yield* readVerification(issueId))?.status).toBe("not_fixed") + }).pipe(Effect.provide(layer)) + }), + ) + + it.effect("keeps the row due when its verdict event cannot commit", () => + Effect.gen(function* () { + const base = createTestDb(createdDbs).layer + const healthy = makeLayer(undefined, base) + const faulty = makeLayer(undefined, failingEventInsertLayer(base)) + + const issueId = yield* Effect.gen(function* () { + const service = yield* IssueFixVerificationService + const id = yield* seedIssue({ severity: "low" }) + yield* service.linkPullRequest(ORG, null, id, PR_URL, "agent") + yield* service.onPullRequestEvent(mergeEvent()) + return id + }).pipe(Effect.provide(healthy)) + + yield* Effect.gen(function* () { + const service = yield* IssueFixVerificationService + const row = yield* readVerification(issueId) + assert.isDefined(row) + const failure = yield* Effect.flip( + service.applyVerdict(row, "verified", "No occurrences.", Date.now()), + ) + assert.strictEqual(failure._tag, "@maple/http/errors/ErrorPersistenceError") + }).pipe(Effect.provide(faulty)) + + // The terminal status must not commit without its event: settled alone, + // the row leaves `waiting`/`running` for good while the issue stays in + // `verifying`, and no later tick can ever pick it back up. + yield* Effect.gen(function* () { + expect((yield* readVerification(issueId))?.status).toBe("waiting") + expect((yield* readIssueState(issueId))?.workflowState).toBe("verifying") + }).pipe(Effect.provide(healthy)) + }), + ) +}) diff --git a/apps/api/src/services/errors/IssueFixVerificationService.ts b/apps/api/src/services/errors/IssueFixVerificationService.ts index 00fb321c7..f8024b0d8 100644 --- a/apps/api/src/services/errors/IssueFixVerificationService.ts +++ b/apps/api/src/services/errors/IssueFixVerificationService.ts @@ -28,6 +28,7 @@ import { } from "@maple/domain/http" import { ErrorIssueId as ErrorIssueIdSchema, type InvestigationId } from "@maple/domain/primitives" import { + errorIssueEvents, errorIssuePullRequests, errorIssues, errorIssueVerifications, @@ -37,7 +38,7 @@ import { type ErrorIssueVerificationRow, } from "@maple/db" import { and, desc, eq, inArray, lte, ne } from "drizzle-orm" -import { Cause, Clock, Context, Effect, Layer, Option, Schema } from "effect" +import { Array as Arr, Cause, Clock, Context, Effect, Layer, Option, Schema } from "effect" import { Database } from "@/platform/DatabaseLive" import { summarizeCause } from "@/platform/describe-cause" import { Env } from "@/platform/Env" @@ -663,23 +664,25 @@ const make: Effect.Effect< // window, and `.returning()` is what says which one this was — a driver // write-result shape would not. const opened = yield* dbExecute((db) => - db.insert(errorIssueVerifications).values({ - id: verificationId, - orgId, - issueId: issue.id, - pullRequestId: link.id, - status: "waiting", - mergedAt: msToDate(mergedAtMs), - verifyAfter: msToDate(verifyAfterMs), - // The builds this issue was known to affect at merge time. Everything - // downstream is a membership test against this array. - baselineVersionsJson: issue.seenVersionsJson, - baselineOccurrenceCount: issue.occurrenceCount, - baselineRatePerHour: ratePerHour, - attempt: 0, - createdAt: msToDate(nowMs), - updatedAt: msToDate(nowMs), - }) + db + .insert(errorIssueVerifications) + .values({ + id: verificationId, + orgId, + issueId: issue.id, + pullRequestId: link.id, + status: "waiting", + mergedAt: msToDate(mergedAtMs), + verifyAfter: msToDate(verifyAfterMs), + // The builds this issue was known to affect at merge time. Everything + // downstream is a membership test against this array. + baselineVersionsJson: issue.seenVersionsJson, + baselineOccurrenceCount: issue.occurrenceCount, + baselineRatePerHour: ratePerHour, + attempt: 0, + createdAt: msToDate(nowMs), + updatedAt: msToDate(nowMs), + }) .onConflictDoNothing() .returning({ id: errorIssueVerifications.id }), ) @@ -736,102 +739,100 @@ const make: Effect.Effect< * this call from the link path the window would never open and the issue would * sit in `in_review` forever. */ - const openMergedVerifications = Effect.fn("IssueFixVerification.openMergedVerifications")( - function* ( - orgId: OrgId, - links: ReadonlyArray, - merge: { - readonly mergedAtMs: number - readonly mergeCommitSha: string | null - readonly nowMs: number - }, + const openMergedVerifications = Effect.fn("IssueFixVerification.openMergedVerifications")(function* ( + orgId: OrgId, + links: ReadonlyArray, + merge: { + readonly mergedAtMs: number + readonly mergeCommitSha: string | null + readonly nowMs: number + }, + ) { + const { mergedAtMs, mergeCommitSha, nowMs } = merge + const systemActor = yield* actors.ensureSystemActor(orgId) + + // One issue's problem must not cost the others. The API doc on + // `onPullRequestEvent` and `openVerification`'s both promise a multi-issue + // delivery does not lose the rest when one issue fails, but only the + // transition/not-found cases were caught — every `dbExecute` in here fails + // outward as `ErrorPersistenceError` and aborted the remaining links, which + // the sink then swallowed, so those links were dropped with no retry and no + // trace of why. + const openForLink = Effect.fn("IssueFixVerification.openForLink")(function* ( + link: (typeof links)[number], ) { - const { mergedAtMs, mergeCommitSha, nowMs } = merge - const systemActor = yield* actors.ensureSystemActor(orgId) - - // One issue's problem must not cost the others. The API doc on - // `onPullRequestEvent` and `openVerification`'s both promise a multi-issue - // delivery does not lose the rest when one issue fails, but only the - // transition/not-found cases were caught — every `dbExecute` in here fails - // outward as `ErrorPersistenceError` and aborted the remaining links, which - // the sink then swallowed, so those links were dropped with no retry and no - // trace of why. - const openForLink = Effect.fn("IssueFixVerification.openForLink")(function* ( - link: (typeof links)[number], - ) { - // One live verification per issue. A `synchronize` after a merge, a - // redelivered webhook, or a second PR attached to the same issue must - // not open a second window. - const open = yield* dbExecute((db) => - db - .select() - .from(errorIssueVerifications) - .where( - and( - eq(errorIssueVerifications.orgId, orgId), - eq(errorIssueVerifications.issueId, link.issueId), - inArray(errorIssueVerifications.status, ["waiting", "running"]), - ), - ) - .limit(1), - ) - if (open[0] !== undefined) return false + // One live verification per issue. A `synchronize` after a merge, a + // redelivered webhook, or a second PR attached to the same issue must + // not open a second window. + const open = yield* dbExecute((db) => + db + .select() + .from(errorIssueVerifications) + .where( + and( + eq(errorIssueVerifications.orgId, orgId), + eq(errorIssueVerifications.issueId, link.issueId), + inArray(errorIssueVerifications.status, ["waiting", "running"]), + ), + ) + .limit(1), + ) + if (Arr.isArrayNonEmpty(open)) return false - const issueRows = yield* dbExecute((db) => - db - .select() - .from(errorIssues) - .where(and(eq(errorIssues.orgId, orgId), eq(errorIssues.id, link.issueId))) - .limit(1), - ) - const issue = issueRows[0] - if (issue === undefined) return false + const issueRows = yield* dbExecute((db) => + db + .select() + .from(errorIssues) + .where(and(eq(errorIssues.orgId, orgId), eq(errorIssues.id, link.issueId))) + .limit(1), + ) + const issue = Arr.head(issueRows) + if (Option.isNone(issue)) return false - yield* workflow.recordEvent(orgId, link.issueId, systemActor.id, "pr_merged", { - payload: { - pullRequestId: link.id, - url: link.url, - repoFullName: link.repoFullName, - number: link.number, - mergeCommitSha, - mergedAt: new Date(mergedAtMs).toISOString(), - }, - timestamp: nowMs, - }) + yield* workflow.recordEvent(orgId, link.issueId, systemActor.id, "pr_merged", { + payload: { + pullRequestId: link.id, + url: link.url, + repoFullName: link.repoFullName, + number: link.number, + mergeCommitSha, + mergedAt: new Date(mergedAtMs).toISOString(), + }, + timestamp: nowMs, + }) - yield* openVerification({ - orgId, - issue, - link: { ...link, mergedAt: msToDate(mergedAtMs), mergeCommitSha }, - mergedAtMs, - nowMs, - systemActor, - }) - return true + yield* openVerification({ + orgId, + issue: issue.value, + link: { ...link, mergedAt: msToDate(mergedAtMs), mergeCommitSha }, + mergedAtMs, + nowMs, + systemActor, }) + return true + }) - let opened = 0 - for (const link of links) { - const didOpen = yield* openForLink(link).pipe( - Effect.catchCause((cause) => - Cause.hasInterruptsOnly(cause) - ? Effect.interrupt - : Effect.logError("[IssueFixVerification] could not open a verification").pipe( - Effect.annotateLogs({ - orgId, - issueId: link.issueId, - pullRequestId: link.id, - error: summarizeCause(cause), - }), - Effect.as(false), - ), - ), - ) - if (didOpen) opened += 1 - } - return opened - }, - ) + let opened = 0 + for (const link of links) { + const didOpen = yield* openForLink(link).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.logError("[IssueFixVerification] could not open a verification").pipe( + Effect.annotateLogs({ + orgId, + issueId: link.issueId, + pullRequestId: link.id, + error: summarizeCause(cause), + }), + Effect.as(false), + ), + ), + ) + if (didOpen) opened += 1 + } + return opened + }) const onPullRequestEvent: IssueFixVerificationServiceApi["onPullRequestEvent"] = Effect.fn( "IssueFixVerification.onPullRequestEvent", @@ -926,9 +927,16 @@ const make: Effect.Effect< updatedAt: msToDate(nowMs), }) .where( - inArray( - errorIssuePullRequests.id, - links.map((link) => link.id), + and( + inArray( + errorIssuePullRequests.id, + links.map((link) => link.id), + ), + // A GitHub merge is irreversible, so a non-merged event reaching a + // link already marked merged can only be a stale or out-of-order + // queue delivery — never let it regress the state or null the merge + // metadata a verification window was opened from. + ...(input.merged ? [] : [ne(errorIssuePullRequests.state, "merged")]), ), ), ) @@ -1031,6 +1039,11 @@ const make: Effect.Effect< const markRunning: IssueFixVerificationServiceApi["markRunning"] = Effect.fn( "IssueFixVerification.markRunning", )(function* (row, investigationId, nowMs) { + // CAS on the state the tick selected the row in: the error tick can refute + // this row while the agent is being enqueued, and resurrecting a terminal + // `not_fixed` back to `running` would overwrite that decisive verdict. The + // orphaned agent run is the cheaper casualty — with no investigationId + // linked, `settledRuns` never picks it up. yield* dbExecute((db) => db .update(errorIssueVerifications) @@ -1039,7 +1052,13 @@ const make: Effect.Effect< investigationId, updatedAt: msToDate(nowMs), }) - .where(eq(errorIssueVerifications.id, row.id)), + .where( + and( + eq(errorIssueVerifications.id, row.id), + eq(errorIssueVerifications.status, row.status), + eq(errorIssueVerifications.attempt, row.attempt), + ), + ), ) }) @@ -1048,6 +1067,22 @@ const make: Effect.Effect< )(function* (row, verdict, note, nowMs) { const systemActor = yield* actors.ensureSystemActor(row.orgId) + // Every status write CASes on the state and attempt the caller read. Two + // writers travel these rows — the error tick's refutation and this path — + // so a stale snapshot must become a no-op, never a `verified` written over + // a decisive `not_fixed`. The verdict event commits in the same + // transaction: a status that settled without its event is invisible to + // the timeline AND to the next tick, so nothing could ever repair it. + const guard = () => + and( + eq(errorIssueVerifications.id, row.id), + eq(errorIssueVerifications.status, row.status), + eq(errorIssueVerifications.attempt, row.attempt), + ) + const lostRace = Effect.logInfo("[FixVerification] verdict lost a race and was not applied").pipe( + Effect.annotateLogs({ orgId: row.orgId, verificationId: row.id, verdict }), + ) + // An inconclusive verdict re-arms one longer window rather than settling. // Past the attempt cap it hands the issue back to a human instead of // looping forever in `verifying`. @@ -1063,48 +1098,49 @@ const make: Effect.Effect< verificationWindowMs({ severity: null, ratePerHour: row.baselineRatePerHour }), firstWindowMs, ) - yield* dbExecute((db) => - db - .update(errorIssueVerifications) - .set({ - status: "waiting", - attempt: row.attempt + 1, - verifyAfter: msToDate(nowMs + extendedMs), - verdict: null, - verdictNote: note, - investigationId: null, - updatedAt: msToDate(nowMs), - }) - .where(eq(errorIssueVerifications.id, row.id)), - ) - yield* workflow.recordEvent(row.orgId, row.issueId, systemActor.id, "verification_verdict", { - payload: { - verificationId: row.id, - verdict: "inconclusive", - note, - retrying: true, - verifyAfter: new Date(nowMs + extendedMs).toISOString(), + const retryEvent = workflow.buildEvent( + row.orgId, + row.issueId, + systemActor.id, + "verification_verdict", + nowMs, + { + payload: { + verificationId: row.id, + verdict: "inconclusive", + note, + retrying: true, + verifyAfter: new Date(nowMs + extendedMs).toISOString(), + }, }, - timestamp: nowMs, - }) + ) + const landed = yield* dbExecute((db) => + db.transaction(async (tx) => { + const updated = await tx + .update(errorIssueVerifications) + .set({ + status: "waiting", + attempt: row.attempt + 1, + verifyAfter: msToDate(nowMs + extendedMs), + verdict: null, + verdictNote: note, + investigationId: null, + updatedAt: msToDate(nowMs), + }) + .where(guard()) + .returning({ id: errorIssueVerifications.id }) + if (updated.length === 0) return false + await tx.insert(errorIssueEvents).values(retryEvent) + return true + }), + ) + if (!landed) yield* lostRace return } const status = verdict === "verified" ? "verified" : verdict === "not_fixed" ? "not_fixed" : "inconclusive" - yield* dbExecute((db) => - db - .update(errorIssueVerifications) - .set({ - status, - verdict, - verdictNote: note, - updatedAt: msToDate(nowMs), - }) - .where(eq(errorIssueVerifications.id, row.id)), - ) - const issueRows = yield* dbExecute((db) => db .select() @@ -1112,23 +1148,51 @@ const make: Effect.Effect< .where(and(eq(errorIssues.orgId, row.orgId), eq(errorIssues.id, row.issueId))) .limit(1), ) - const issue = issueRows[0] - if (issue === undefined) return + const issue = Arr.head(issueRows) - const autoCloses = verdict === "verified" && verificationVerdictAutoCloses(issue.severity ?? null) + const autoCloses = + Option.isSome(issue) && + verdict === "verified" && + verificationVerdictAutoCloses(issue.value.severity ?? null) - yield* workflow.recordEvent(row.orgId, row.issueId, systemActor.id, "verification_verdict", { - payload: { - verificationId: row.id, - verdict, - note, - autoClosed: autoCloses, - severity: issue.severity ?? null, - postMergeOccurrenceCount: row.postMergeOccurrenceCount, - investigationId: row.investigationId, - }, - timestamp: nowMs, - }) + // A vanished issue still gets its verdict on the verification row; there is + // just no timeline to write the event to. + const verdictEvent = Option.map(issue, (issue) => + workflow.buildEvent(row.orgId, row.issueId, systemActor.id, "verification_verdict", nowMs, { + payload: { + verificationId: row.id, + verdict, + note, + autoClosed: autoCloses, + severity: issue.severity ?? null, + postMergeOccurrenceCount: row.postMergeOccurrenceCount, + investigationId: row.investigationId, + }, + }), + ) + + const landed = yield* dbExecute((db) => + db.transaction(async (tx) => { + const updated = await tx + .update(errorIssueVerifications) + .set({ + status, + verdict, + verdictNote: note, + updatedAt: msToDate(nowMs), + }) + .where(guard()) + .returning({ id: errorIssueVerifications.id }) + if (updated.length === 0) return false + if (Option.isSome(verdictEvent)) await tx.insert(errorIssueEvents).values(verdictEvent.value) + return true + }), + ) + if (!landed) { + yield* lostRace + return + } + if (Option.isNone(issue)) return // `verified` on a high/critical issue deliberately transitions nothing: the // verdict and its evidence are on the timeline, and a human closes it. See @@ -1144,7 +1208,7 @@ const make: Effect.Effect< if (target === null) return yield* workflow - .applyTransition(row.orgId, systemActor.id, issue, target, { + .applyTransition(row.orgId, systemActor.id, issue.value, target, { payload: { viaVerification: row.id, verdict }, timestamp: nowMs, }) @@ -1154,7 +1218,7 @@ const make: Effect.Effect< Effect.logInfo("[FixVerification] verdict could not move the issue").pipe( Effect.annotateLogs({ issueId: row.issueId, - from: issue.workflowState, + from: issue.value.workflowState, to: target, reason: error.message, }), diff --git a/apps/api/src/services/errors/fix-verification-enqueue.test.ts b/apps/api/src/services/errors/fix-verification-enqueue.test.ts index 81b885693..c64b699d9 100644 --- a/apps/api/src/services/errors/fix-verification-enqueue.test.ts +++ b/apps/api/src/services/errors/fix-verification-enqueue.test.ts @@ -2,11 +2,7 @@ import { randomUUID } from "node:crypto" import { afterEach, assert, describe, it } from "@effect/vitest" import { Clock, ConfigProvider, Effect, Layer, Schema } from "effect" import { OrgId } from "@maple/domain/http" -import { - ErrorIssueId, - ErrorIssuePullRequestId, - ErrorIssueVerificationId, -} from "@maple/domain/primitives" +import { ErrorIssueId, ErrorIssuePullRequestId, ErrorIssueVerificationId } from "@maple/domain/primitives" import { aiTriageSettings, errorIssues, @@ -167,6 +163,10 @@ describe("enqueueFixVerification", () => { ) assert.strictEqual(rows.length, 1) assert.strictEqual(rows[0]?.status, "investigating") + // The fence a restart needs: `restartInvestigation` terminates the prior + // workflow only when this column is populated. Left null, the old + // instance survives every restart and publishes over the new attempt. + assert.strictEqual(rows[0]?.workflowInstanceId, workflow.created[0]?.id) }).pipe(Effect.provide(makeLayer())), ) diff --git a/apps/api/src/services/errors/investigation-fanout-start.ts b/apps/api/src/services/errors/investigation-fanout-start.ts index c4e17a6ff..187c4134d 100644 --- a/apps/api/src/services/errors/investigation-fanout-start.ts +++ b/apps/api/src/services/errors/investigation-fanout-start.ts @@ -90,6 +90,21 @@ export const startInvestigationFanout: ( return { started: false, reason: "no_binding" as const } } + // Persist the instance id BEFORE dispatch, exactly as the manual start path + // does. Without it every automatically created investigation kept a null + // `workflowInstanceId`, and `restartInvestigation` — which terminates the + // prior instance only when the column is populated — left the old workflow + // running to publish over the replacement attempt. Attempt 0's instance id + // is deterministic: the bare investigation id. + yield* database + .execute((db) => + db + .update(investigations) + .set({ workflowInstanceId: investigationId, updatedAt: new Date(nowMs) }) + .where(eq(investigations.id, investigationId)), + ) + .pipe(Effect.asVoid) + // `Exit`, not `Effect.option`: the reason a create() failed is the whole // diagnostic value here — an id collision means a live instance already owns // this investigation, a network error means retry. diff --git a/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts b/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts index f79b089dd..cd6e7e916 100644 --- a/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts +++ b/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts @@ -18,7 +18,7 @@ import { eq } from "drizzle-orm" import { encryptAes256Gcm, parseBase64Aes256GcmKey } from "@/platform/Crypto" import { Database } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" -import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { cleanupTestDbs, createTestDb, executeSql, queryFirstRow, type TestDb } from "@/platform/test-pglite" import { WarehouseQueryService, type WarehouseQueryServiceApi, @@ -205,6 +205,10 @@ interface FetchOptions { /** Live Worker scripts the REST enumeration returns (default: my-worker). */ readonly workerScripts?: ReadonlyArray<{ id: string }> /** Hyperdrive configs the REST enumeration returns (default: none). Mutable so a test can change the upstream set between polls. */ + /** Serve this many synthetic active zones with real pagination (50/page). */ + zonesTotal?: number + /** Awaited before answering each GraphQL call — lets a test interleave a concurrent write. */ + onGraphql?: () => Promise hyperdriveConfigs?: ReadonlyArray /** HTTP status for the hyperdrive listing (default 200). Mutable for per-poll failure injection. */ hyperdriveStatus?: number @@ -255,6 +259,7 @@ const mockCloudflareFetch = return jsonResponse({}, options.metricsStatus ?? 200) } if (url.includes("/graphql")) { + await options.onGraphql?.() const body = JSON.parse(await readRequestBody(input, init)) as { query: string } options.graphqlQueries?.push(body.query) if (options.graphqlErrors) { @@ -308,6 +313,23 @@ const mockCloudflareFetch = ) } const page = Number(new URL(url).searchParams.get("page") ?? "1") + if (options.zonesTotal != null) { + const perPage = 50 + const start = (page - 1) * perPage + const count = Math.min(perPage, Math.max(0, options.zonesTotal - start)) + const synthetic = Array.from({ length: count }, (_, index) => ({ + ...zoneFixture, + id: `zone-${start + index}`, + name: `z${start + index}.example.com`, + })) + return jsonResponse({ + success: true, + errors: [], + messages: [], + result: synthetic, + result_info: { count, page, per_page: perPage, total_count: options.zonesTotal }, + }) + } const forAccount = options.zonesByAccount == null ? null @@ -894,6 +916,34 @@ describe("CloudflareAnalyticsService", () => { }).pipe(Effect.provide(makeLayer(testDb, captured))) }) + it.effect("pollOrg does NOT disable unseen zones when the zone listing is truncated", () => { + const testDb = createTestDb(trackedDbs) + const captured: CapturedIngest[] = [] + return Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection() + // A zone the account still owns, but which fell past the 200-zone + // discovery cap of a 201-zone listing: unseen, not removed. + yield* seedStateRow({ + dataset: "http_requests", + zoneId: "beyond-cap-zone", + zoneName: "beyond.example.com", + watermarkAt: new Date(T0 - 20 * MIN), + settingsFetchedAt: new Date(T0 - 5 * MIN), + }) + const service = yield* CloudflareAnalyticsService + yield* service.pollOrg(ORG) + const rows = yield* loadStateRows + // A vanished row fails the test through the error channel, so the + // assertions below get a narrowed row instead of a `!`. + const beyond = yield* Effect.fromOption( + Arr.findFirst(rows, (row) => row.zoneId === "beyond-cap-zone"), + ) + assert.isTrue(beyond.enabled) + assert.notInclude(beyond.lastError ?? "", "no longer present") + }).pipe(Effect.provide(makeLayer(testDb, captured, { zonesTotal: 201 }))) + }) + it.effect("pollOrg skips when another tick holds the lease", () => { const testDb = createTestDb(trackedDbs) const captured: CapturedIngest[] = [] @@ -908,6 +958,50 @@ describe("CloudflareAnalyticsService", () => { }).pipe(Effect.provide(makeLayer(testDb, captured))) }) + it.effect("a tick that lost its lease does not clear the successor's on release", () => { + const testDb = createTestDb(trackedDbs) + const captured: CapturedIngest[] = [] + // Simulate the overlap: while this tick is mid-poll (first GraphQL call), + // a successor claims the anchor lease — as happens when a tick outlives + // LEASE_MS. The finishing tick's release must be a compare-and-set on the + // lease value it wrote, so the successor's live lease survives. + const SUCCESSOR_LEASE = new Date(T0 + 3 * MIN) + let injected = false + const onGraphql = async () => { + if (injected) return + injected = true + await executeSql( + testDb, + `UPDATE cloudflare_analytics_state SET lease_until = $1 + WHERE dataset = 'workers_invocations' AND zone_id = ''`, + [SUCCESSOR_LEASE], + ) + } + return Effect.gen(function* () { + yield* TestClock.setTime(T0) + yield* seedConnection() + yield* seedStateRow({ + dataset: "workers_invocations", + zoneId: "", + discoveredAt: new Date(T0 - 5 * MIN), + settingsFetchedAt: new Date(T0 - 5 * MIN), + }) + const service = yield* CloudflareAnalyticsService + yield* service.pollOrg(ORG) + + const anchor = yield* Effect.promise(() => + queryFirstRow<{ lease_until: Date | string | null }>( + testDb, + `SELECT lease_until FROM cloudflare_analytics_state + WHERE dataset = 'workers_invocations' AND zone_id = ''`, + ), + ) + // Pre-CAS this was nulled by the stale tick's unconditional release, + // letting a third tick overlap the successor. + assert.strictEqual(new Date(anchor?.lease_until ?? 0).getTime(), SUCCESSOR_LEASE.getTime()) + }).pipe(Effect.provide(makeLayer(testDb, captured, { onGraphql }))) + }) + it.effect("pollOrg reclaims a far-future (corrupt) lease instead of skipping forever", () => { const testDb = createTestDb(trackedDbs) const captured: CapturedIngest[] = [] diff --git a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts index cf531738e..1a33cc1d5 100644 --- a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts +++ b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts @@ -21,8 +21,10 @@ * `settings` (Cloudflare's per-plan limits), a lease column as the tick-overlap guard, and * last success/error for the integration UI. The * poll loop is resumable by construction — a budget-exhausted backfill simply continues from - * its watermark on the next tick. Metrics rows are written exactly once because a window is - * only ever ingested before its watermark advances, and windows never overlap. + * its watermark on the next tick. Metrics delivery is at-least-once: a window is only ever + * ingested before its watermark advances and windows never overlap, so steady state writes each + * row once — but a crash between the gateway accepting a batch and the frontier write landing + * replays that window next tick (the warehouse has no dedupe key to absorb it). */ import { randomUUID } from "node:crypto" import { @@ -1275,8 +1277,18 @@ export class CloudflareAnalyticsService extends Context.Service< * Release the tick lease. `holdUntilMs` keeps it held instead of clearing it, which is how a * rate-limited org sits out the next tick(s) — `claimLease` already refuses a live lease, and * a hold under `2 * LEASE_MS` stays inside its corrupt-lease escape hatch. + * + * Compare-and-set on `claimedUntil` (the value this tick's claim wrote): a tick that + * outlived its own lease must not clear the successor's — releasing the successor's live + * lease would let a third tick overlap it and replay the same windows into the gateway. */ - const releaseLease = (orgId: OrgId, accountId: string, now: number, holdUntilMs?: number) => + const releaseLease = ( + orgId: OrgId, + accountId: string, + now: number, + claimedUntil: Date | null, + holdUntilMs?: number, + ) => dbExecute((db) => db .update(cloudflareAnalyticsState) @@ -1290,6 +1302,9 @@ export class CloudflareAnalyticsService extends Context.Service< eq(cloudflareAnalyticsState.accountId, accountId), eq(cloudflareAnalyticsState.dataset, WORKERS_DATASET), eq(cloudflareAnalyticsState.zoneId, ""), + claimedUntil == null + ? isNull(cloudflareAnalyticsState.leaseUntil) + : eq(cloudflareAnalyticsState.leaseUntil, claimedUntil), ), ), ) @@ -1304,6 +1319,7 @@ export class CloudflareAnalyticsService extends Context.Service< zones: ReadonlyArray, anchorId: string, now: number, + zonesTruncated: boolean, ) { const rows = yield* loadStateRows(orgId, accountId) // Every zone-scoped dataset gets one state row per discovered zone — reconcile them all. @@ -1357,11 +1373,17 @@ export class CloudflareAnalyticsService extends Context.Service< // `rows` is already scoped to this connection's account, so a zone that moved to a // sibling account is disabled here and freshly discovered by that account's tick. + // Never off a truncated listing: a zone past the discovery cap was unseen, not + // removed, and disabling it would silently stop its telemetry. const seen = new Set(zones.map((zone) => zone.id)) - const vanished = rows.filter( - (row) => - DATASET_BY_ID.get(row.dataset)?.scope === "zone" && row.enabled && !seen.has(row.zoneId), - ) + const vanished = zonesTruncated + ? [] + : rows.filter( + (row) => + DATASET_BY_ID.get(row.dataset)?.scope === "zone" && + row.enabled && + !seen.has(row.zoneId), + ) yield* updateRows( vanished.map((row) => row.id), { @@ -1784,6 +1806,7 @@ export class CloudflareAnalyticsService extends Context.Service< accountId: string, configs: ReadonlyArray, now: number, + configsTruncated: boolean, ) { yield* Effect.annotateCurrentSpan({ orgId, @@ -1837,9 +1860,11 @@ export class CloudflareAnalyticsService extends Context.Service< // Soft-delete only THIS account's vanished configs — a sibling account's inventory // is invisible to this listing and must not be reaped by it. Pre-multi-account rows // (null accountId) are adopted by whichever account's reconcile still sees them. + // Never off a truncated listing: a config past the cap was unseen, not removed. yield* Effect.forEach( existingRows.filter( (row) => + !configsTruncated && !upstreamIds.has(row.configId) && row.deletedAt === null && (row.accountId == null || row.accountId === accountId), @@ -1977,7 +2002,14 @@ export class CloudflareAnalyticsService extends Context.Service< // Newly-registered account datasets get their rows on the discovery cadence — // before reconcileZones, whose loadStateRows picks them up for this tick. yield* ensureAccountRows(orgId, accountId, now) - rows = yield* reconcileZones(orgId, accountId, zonesResult.success, anchor.id, now) + rows = yield* reconcileZones( + orgId, + accountId, + zonesResult.success.items, + anchor.id, + now, + zonesResult.success.truncated, + ) // Script enumeration rides the same discovery TTL. It filters deleted scripts out of // the workers dataset; a failure (typically a pre-workers-scripts.read grant) degrades // open — emit everything rather than wedge the org, and keep the last known set. @@ -1990,10 +2022,21 @@ export class CloudflareAnalyticsService extends Context.Service< errorTag: scriptsResult.failure._tag, error: scriptsResult.failure.message, }) + } else if (scriptsResult.success.truncated) { + // More live scripts exist than the enumeration cap. An incomplete + // membership set would silently drop valid Worker metrics, so + // degrade open: no filter, and clear the persisted set so later + // ticks don't filter on a stale one either. + yield* Effect.logWarning("cloudflare-analytics script enumeration truncated", { + orgId, + scriptCount: scriptsResult.success.items.length, + }) + liveScripts = null + yield* updateRows([anchor.id], { liveScriptsJson: null, updatedAt: msToDate(now) }) } else { - liveScripts = new Set(scriptsResult.success) + liveScripts = new Set(scriptsResult.success.items) yield* updateRows([anchor.id], { - liveScriptsJson: JSON.stringify(scriptsResult.success), + liveScriptsJson: JSON.stringify(scriptsResult.success.items), updatedAt: msToDate(now), }) } @@ -2011,7 +2054,13 @@ export class CloudflareAnalyticsService extends Context.Service< error: hyperdriveResult.failure.message, }) } else { - yield* reconcileHyperdriveConfigs(orgId, accountId, hyperdriveResult.success, now) + yield* reconcileHyperdriveConfigs( + orgId, + accountId, + hyperdriveResult.success.items, + now, + hyperdriveResult.success.truncated, + ) } } else { rows = yield* loadStateRows(orgId, accountId) @@ -2210,6 +2259,7 @@ export class CloudflareAnalyticsService extends Context.Service< orgId, accountId, end, + anchor.leaseUntil, rateLimited ? end + RATE_LIMIT_BACKOFF_MS : undefined, ).pipe( // The ensuring must never fail (that would mask whatever this tick actually diff --git a/apps/api/src/services/integrations/CloudflareApi.ts b/apps/api/src/services/integrations/CloudflareApi.ts index dcc3009b2..52c45600b 100644 --- a/apps/api/src/services/integrations/CloudflareApi.ts +++ b/apps/api/src/services/integrations/CloudflareApi.ts @@ -14,7 +14,11 @@ import { Effect } from "effect" import type * as Impl from "./CloudflareApiImpl" -export type { CloudflareGraphqlError, CloudflareHyperdriveConfig, CloudflareZone } from "./CloudflareApiImpl" +export type { + CloudflareGraphqlError, + CloudflareHyperdriveConfig, + CloudflareZone, +} from "./CloudflareApiImpl" const impl = Effect.promise(() => import("./CloudflareApiImpl")) diff --git a/apps/api/src/services/integrations/CloudflareApiImpl.ts b/apps/api/src/services/integrations/CloudflareApiImpl.ts index 0e04edd95..e3a787d86 100644 --- a/apps/api/src/services/integrations/CloudflareApiImpl.ts +++ b/apps/api/src/services/integrations/CloudflareApiImpl.ts @@ -22,7 +22,7 @@ import * as Hyperdrive from "@distilled.cloud/cloudflare/hyperdrive" import * as Workers from "@distilled.cloud/cloudflare/workers" import * as Zones from "@distilled.cloud/cloudflare/zones" import { IntegrationsRevokedError, IntegrationsUpstreamError } from "@maple/domain/http" -import { Effect, Layer, Schema, Stream } from "effect" +import { Effect, Layer, Match, Schema, Stream } from "effect" import { FetchHttpClient, type HttpClient } from "effect/unstable/http" /** The Effect context a distilled operation requires: resolved credentials + an HTTP client. */ @@ -145,6 +145,29 @@ export interface CloudflareZone { // pathological account with thousands of zones must not fan out unbounded work. const MAX_ZONES = 200 +/** + * A bounded listing. `truncated` means the cap was hit and more items exist + * upstream — callers must not treat `items` as a complete inventory (no + * disabling/deleting/filtering of resources that merely fell past the cap). + */ +export interface BoundedListing { + readonly items: ReadonlyArray + readonly truncated: boolean +} + +/** Collect up to `max` items, peeking one past the cap so truncation is observable. */ +const collectBounded = (stream: Stream.Stream, max: number) => + stream.pipe( + Stream.take(max + 1), + Stream.runCollect, + Effect.map( + (collected): BoundedListing => ({ + items: collected.slice(0, max), + truncated: collected.length > max, + }), + ), + ) + /** * List the account's active zones. Used by the analytics poller for zone discovery — each active * zone gets a poll-state row (and thus edge metrics under `cloudflare/{zoneName}`). @@ -153,23 +176,28 @@ export const listZones: ( accessToken: string, accountId: string, apiBaseUrl?: string, -) => Effect.Effect, CloudflareApiError, never> = Effect.fn( +) => Effect.Effect, CloudflareApiError, never> = Effect.fn( "CloudflareApi.listZones", )(function* (accessToken: string, accountId: string, apiBaseUrl?: string) { yield* Effect.annotateCurrentSpan("maple.cloudflare.account_id", accountId) const zones = yield* runMapped( accessToken, - Zones.listZones - .items({ account: { id: accountId }, status: "active", perPage: 50 }) - .pipe(Stream.take(MAX_ZONES), Stream.runCollect), + collectBounded( + Zones.listZones.items({ account: { id: accountId }, status: "active", perPage: 50 }), + MAX_ZONES, + ), apiBaseUrl, ) - yield* Effect.annotateCurrentSpan("maple.cloudflare.zone_count", zones.length) - return zones.map((zone) => ({ - id: zone.id, - name: zone.name, - status: zone.status ?? null, - })) + yield* Effect.annotateCurrentSpan("maple.cloudflare.zone_count", zones.items.length) + yield* Effect.annotateCurrentSpan("maple.cloudflare.zone_listing_truncated", zones.truncated) + return { + items: zones.items.map((zone) => ({ + id: zone.id, + name: zone.name, + status: zone.status ?? null, + })), + truncated: zones.truncated, + } }) // Script enumeration is bounded like zone discovery: the poller only needs a membership set to @@ -185,17 +213,21 @@ export const listWorkerScripts: ( accessToken: string, accountId: string, apiBaseUrl?: string, -) => Effect.Effect, CloudflareApiError, never> = Effect.fn( +) => Effect.Effect, CloudflareApiError, never> = Effect.fn( "CloudflareApi.listWorkerScripts", )(function* (accessToken: string, accountId: string, apiBaseUrl?: string) { yield* Effect.annotateCurrentSpan("maple.cloudflare.account_id", accountId) const scripts = yield* runMapped( accessToken, - Workers.listScripts.items({ accountId }).pipe(Stream.take(MAX_SCRIPTS), Stream.runCollect), + collectBounded(Workers.listScripts.items({ accountId }), MAX_SCRIPTS), apiBaseUrl, ) - yield* Effect.annotateCurrentSpan("maple.cloudflare.script_count", scripts.length) - return scripts.flatMap((script) => (script.id == null || script.id === "" ? [] : [script.id])) + yield* Effect.annotateCurrentSpan("maple.cloudflare.script_count", scripts.items.length) + yield* Effect.annotateCurrentSpan("maple.cloudflare.script_listing_truncated", scripts.truncated) + return { + items: scripts.items.flatMap((script) => (script.id == null || script.id === "" ? [] : [script.id])), + truncated: scripts.truncated, + } }) export interface CloudflareHyperdriveConfig { @@ -222,33 +254,54 @@ const MAX_HYPERDRIVE_CONFIGS = 200 * database — e.g. a PlanetScale database). The `origin` union (standard / Access-client / VPC * service) is normalized to nullable host/port. */ +// The SDK's origin union has no discriminant, so each arm matches its variant's +// distinguishing field: only a public origin carries `port`, an Access origin a +// client id, a VPC origin only a service id. `Match.exhaustive` makes a fourth +// variant a compile error instead of a silently null host/port. +const normalizeHyperdriveOrigin = ( + origin: Hyperdrive.ConfigsListResultItemOrigin, +): { readonly host: string | null; readonly port: number | null } => + Match.value(origin).pipe( + Match.when({ port: Match.number }, (publicOrigin) => ({ + host: publicOrigin.host, + port: publicOrigin.port, + })), + Match.when({ accessClientId: Match.string }, (accessOrigin) => ({ + host: accessOrigin.host, + port: null, + })), + Match.when({ serviceId: Match.string }, () => ({ host: null, port: null })), + Match.exhaustive, + ) + export const listHyperdriveConfigs: ( accessToken: string, accountId: string, apiBaseUrl?: string, -) => Effect.Effect, CloudflareApiError, never> = Effect.fn( +) => Effect.Effect, CloudflareApiError, never> = Effect.fn( "CloudflareApi.listHyperdriveConfigs", )(function* (accessToken: string, accountId: string, apiBaseUrl?: string) { yield* Effect.annotateCurrentSpan("maple.cloudflare.account_id", accountId) const configs = yield* runMapped( accessToken, - Hyperdrive.listConfigs - .items({ accountId }) - .pipe(Stream.take(MAX_HYPERDRIVE_CONFIGS), Stream.runCollect), + collectBounded(Hyperdrive.listConfigs.items({ accountId }), MAX_HYPERDRIVE_CONFIGS), apiBaseUrl, ) - yield* Effect.annotateCurrentSpan("maple.cloudflare.hyperdrive_config_count", configs.length) - return configs.map((config) => ({ - id: config.id, - name: config.name, - origin: { - host: "host" in config.origin ? config.origin.host : null, - port: "port" in config.origin ? config.origin.port : null, - database: config.origin.database, - scheme: config.origin.scheme, - user: config.origin.user ?? null, - }, - })) + yield* Effect.annotateCurrentSpan("maple.cloudflare.hyperdrive_config_count", configs.items.length) + yield* Effect.annotateCurrentSpan("maple.cloudflare.hyperdrive_listing_truncated", configs.truncated) + return { + items: configs.items.map((config) => ({ + id: config.id, + name: config.name, + origin: { + ...normalizeHyperdriveOrigin(config.origin), + database: config.origin.database, + scheme: config.origin.scheme, + user: config.origin.user ?? null, + }, + })), + truncated: configs.truncated, + } }) // GraphQL Analytics (the raw escape hatch the module doc-comment anticipates) diff --git a/apps/api/src/services/integrations/PlanetScaleService.test.ts b/apps/api/src/services/integrations/PlanetScaleService.test.ts index 7f4b9b172..2ae5f4f22 100644 --- a/apps/api/src/services/integrations/PlanetScaleService.test.ts +++ b/apps/api/src/services/integrations/PlanetScaleService.test.ts @@ -1,5 +1,5 @@ import { afterEach, assert, describe, it } from "@effect/vitest" -import { ConfigProvider, Effect, Fiber, Layer, Schema } from "effect" +import { ConfigProvider, Effect, Fiber, Layer, Predicate, Schema } from "effect" import { TestClock } from "effect/testing" import { FetchHttpClient } from "effect/unstable/http" import { OrgId, UserId } from "@maple/domain/http" @@ -1002,3 +1002,145 @@ describe("deployRequestTimelineRows", () => { assert.deepStrictEqual(rows, []) }) }) + +describe("PlanetScaleService pagination truncation", () => { + /** + * Serve `total` synthetic items 100 per page so `fetchAllPages` sees ten + * FULL pages and stops at its ceiling with no short page — a truncated + * listing. Everything else behaves like `stubApi`. + */ + const stubTruncated = (options: { + totalDatabases?: number + deployRequestsTotalFor?: string + deployRequestUpdatedAt?: string + }) => { + const base = stubApi({ databases: [], branchesByDatabase: {} }) + const pageOf = (url: string) => { + const match = url.match(/[?&]page=(\d+)/) + return match ? Number(match[1]) : 1 + } + const stub = (async (input: string | URL | Request, init?: RequestInit) => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url + const json = (body: unknown) => + new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }) + if ( + Predicate.isNotUndefined(options.deployRequestsTotalFor) && + url.includes("/deploy-requests") + ) { + const page = pageOf(url) + const start = (page - 1) * 100 + return json({ + data: Array.from({ length: 100 }, (_, index) => ({ + id: `dr_${start + index}`, + number: start + index, + state: "closed", + deployment_state: "complete", + created_at: options.deployRequestUpdatedAt, + updated_at: options.deployRequestUpdatedAt, + closed_at: options.deployRequestUpdatedAt, + })), + }) + } + if ( + Predicate.isNotUndefined(options.totalDatabases) && + /\/databases(\?|$)/.test(url.split("#")[0] ?? url) + ) { + const page = pageOf(url) + const start = (page - 1) * 100 + const count = Math.min(100, Math.max(0, options.totalDatabases - start)) + return json({ + data: Array.from({ length: count }, (_, index) => ({ + id: `db_${start + index}`, + name: `db-${start + index}`, + })), + }) + } + return base(input, init) + }) as typeof fetch + globalThis.fetch = stub + return stub + } + + it.effect( + "does not soft-delete stored databases when the inventory listing is truncated", + () => { + const testDb = createTestDb(trackedDbs) + // Ten full pages: the upstream org has MORE than 1,000 databases, and + // the stored one may simply live beyond the pagination ceiling. + const stub = stubTruncated({ totalDatabases: 1000 }) + return Effect.gen(function* () { + yield* connect("org_1") + yield* Effect.promise(() => + executeSql( + testDb, + `INSERT INTO planetscale_databases (id, org_id, database_id, name, kind, created_at, updated_at) + VALUES ('row_beyond', 'org_1', 'db_beyond_cap', 'beyond-cap', 'mysql', now(), now())`, + ), + ) + const service = yield* PlanetScaleService + const summary = yield* service.pollAllOrgs() + assert.strictEqual(summary.refreshed, 1) + + const kept = yield* Effect.promise(() => + queryFirstRow<{ deleted_at: string | null }>( + testDb, + "SELECT deleted_at FROM planetscale_databases WHERE database_id = 'db_beyond_cap'", + ), + ) + assert.isNull(kept?.deleted_at) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide(Layer.mergeAll(makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub))), + ) + }, + 60_000, + ) + + it.effect("does not advance the deploy-request watermark past a truncated listing", () => { + const testDb = createTestDb(trackedDbs) + // All requests sit past the 30-day floor so no timeline rows are written; + // the only observable effect is the watermark decision. + const stub = stubTruncated({ + deployRequestsTotalFor: "main-db", + deployRequestUpdatedAt: "2026-01-01T00:00:00.000Z", + }) + return Effect.gen(function* () { + yield* connect("org_1") + yield* Effect.promise(() => + executeSql( + testDb, + `INSERT INTO planetscale_databases (id, org_id, database_id, name, kind, created_at, updated_at) + VALUES ('row_main', 'org_1', 'db_main', 'main-db', 'mysql', now(), now())`, + ), + ) + const service = yield* PlanetScaleService + // Move the TestClock well past the fixture timestamps so they parse as + // real past instants that sit beyond the 30-day backfill floor (no + // timeline rows, but a positive newest-update candidate). + yield* TestClock.adjust("500000 hours") + const summary = yield* service.pollAllOrgs() + // The poll itself succeeded — a decode failure would also leave the + // watermark null and make this test vacuous. + assert.strictEqual(summary.failures, 0) + + // Ten full pages of deploy requests were read and the ceiling hit: + // requests beyond it were never observed, so the watermark must not + // claim them as done. + const state = yield* Effect.promise(() => + queryFirstRow<{ watermark_at: string | null }>( + testDb, + `SELECT watermark_at FROM planetscale_poll_state + WHERE org_id = 'org_1' AND dataset = 'deploy_requests' AND database_id = 'db_main'`, + ), + ) + assert.isNull(state?.watermark_at) + }).pipe( + Effect.provideService(FetchHttpClient.Fetch, stub), + Effect.provide(Layer.mergeAll(makeLayer(testDb), Layer.succeed(FetchHttpClient.Fetch, stub))), + ) + }) +}) diff --git a/apps/api/src/services/integrations/PlanetScaleService.ts b/apps/api/src/services/integrations/PlanetScaleService.ts index d6e10f479..b361865b6 100644 --- a/apps/api/src/services/integrations/PlanetScaleService.ts +++ b/apps/api/src/services/integrations/PlanetScaleService.ts @@ -21,7 +21,7 @@ import { type PlanetScaleEventRow, } from "@maple/db" import { and, desc, eq, gte, inArray, isNull, lt, lte, or } from "drizzle-orm" -import { Cause, Clock, Context, Duration, Effect, Layer, Schedule, Schema } from "effect" +import { Cause, Clock, Context, Duration, Effect, Layer, Predicate, Schedule, Schema } from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http" import { Database } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" @@ -365,12 +365,19 @@ export class PlanetScaleService extends Context.Service(basePath: string, authorization: string, itemSchema: S) { const decodePage = Schema.decodeUnknownEffect(Schema.fromJsonString(PageSchema(itemSchema))) const items: Array = [] + let complete = false for (let page = 1; page <= MAX_PAGES; page++) { const separator = basePath.includes("?") ? "&" : "?" const response = yield* apiGetJson( @@ -405,9 +412,19 @@ export class PlanetScaleService extends Context.Service Effect.gen(function* () { - const branches = yield* fetchAllPages( + const { items: branches } = yield* fetchAllPages( `/v1/organizations/${org}/databases/${encodeURIComponent(db.name)}/branches`, authorization, BranchSchema, @@ -735,9 +752,15 @@ export class PlanetScaleService extends Context.Service !upstreamIds.has(row.databaseId) && row.deletedAt === null), + existingRows.filter( + (row) => + inventoryComplete && + !upstreamIds.has(row.databaseId) && + Predicate.isNull(row.deletedAt), + ), (row) => database .execute((client) => @@ -789,7 +812,7 @@ export class PlanetScaleService extends Context.Service 0 + ? new Date(newestUpdate) + : null + : (state?.watermarkAt ?? null) yield* recordPollResult( connection.orgId, DEPLOY_REQUESTS_DATASET, database_.databaseId, null, - newestUpdate > 0 ? new Date(newestUpdate) : null, + watermark, ) yield* Effect.annotateCurrentSpan({ "maple.planetscale.deploy_requests.seen": requests.length, diff --git a/apps/api/src/services/integrations/SlackIntegrationService.test.ts b/apps/api/src/services/integrations/SlackIntegrationService.test.ts index 61c060615..13a274909 100644 --- a/apps/api/src/services/integrations/SlackIntegrationService.test.ts +++ b/apps/api/src/services/integrations/SlackIntegrationService.test.ts @@ -1822,6 +1822,83 @@ describe("SlackIntegrationService", () => { assert.strictEqual(again.revoked, false) }).pipe(Effect.provide(withFetch(testDb, neverFetch))) }) + + it.effect("loses gracefully to a concurrent reinstall instead of revoking it", () => { + const testDb = createTestDb(trackedDbs) + const FRESH_KEY_ID = "99999999-8888-4777-8666-555555555555" + const arm = { active: false } + // Interpose on Database so the very next execute after arming — the + // revoke's snapshot SELECT — is immediately followed by a "concurrent + // completeInstall" landing fresh secrets and a new API key on the row. + const racingDb = Layer.effect( + Database, + Effect.gen(function* () { + const real = yield* Database + return { + execute: (fn) => + real.execute(fn).pipe( + Effect.tap(() => { + if (!arm.active) return Effect.void + arm.active = false + return Effect.promise(() => + executeSql( + testDb, + `UPDATE slack_workspaces + SET updated_at = updated_at + interval '1 second', + api_key_id = $2, + bot_token_ciphertext = 'fresh-bot-ciphertext' + WHERE team_id = $1`, + ["T-RACE", FRESH_KEY_ID], + ), + ) + }), + ), + } + }), + ).pipe(Layer.provide(testDb.layer)) + const serviceLayer = SlackIntegrationService.layer.pipe( + Layer.provide(Layer.mergeAll(ApiKeysService.layer, OAuthStateRepository.layer)), + Layer.provide(racingDb), + Layer.provide(Env.layer), + Layer.provide(makeConfig(true)), + ) + return Effect.gen(function* () { + yield* Effect.promise(() => + insertWorkspace(testDb, { + id: "sw_race", + orgId: "org_race", + teamId: "T-RACE", + teamName: "RaceOrg", + botToken: "xoxb-race", + apiKey: "maple_ak_race", + }), + ) + const slack = yield* SlackIntegrationService + arm.active = true + const result = yield* slack.revokeByTeamId("T-RACE", "tokens_revoked") + // The stale revocation must NOT clobber the reinstall: it reports the + // lost race and leaves the fresh row active with its secrets intact. + assert.strictEqual(result.revoked, false) + + const row = yield* Effect.promise(() => + queryFirstRow<{ + revoked_at: string | null + api_key_id: string | null + bot_token_ciphertext: string | null + }>( + testDb, + "SELECT revoked_at, api_key_id, bot_token_ciphertext FROM slack_workspaces WHERE team_id = 'T-RACE'", + ), + ) + assert.isNull(row?.revoked_at) + assert.strictEqual(row?.api_key_id, FRESH_KEY_ID) + assert.strictEqual(row?.bot_token_ciphertext, "fresh-bot-ciphertext") + }).pipe( + Effect.provide( + Layer.mergeAll(serviceLayer, Layer.succeed(FetchHttpClient.Fetch, neverFetch)), + ), + ) + }) }) describe("reconcileWorkspaces", () => { diff --git a/apps/api/src/services/integrations/SlackIntegrationService.ts b/apps/api/src/services/integrations/SlackIntegrationService.ts index 871c9a909..9a568b6f9 100644 --- a/apps/api/src/services/integrations/SlackIntegrationService.ts +++ b/apps/api/src/services/integrations/SlackIntegrationService.ts @@ -1296,18 +1296,14 @@ const make: Effect.Effect< ) yield* Effect.annotateCurrentSpan({ orgId }) const now = yield* Clock.currentTimeMillis - // Revoke the minted API key (best-effort — bookkeeping must not fail the - // revoke). Unlike `uninstall`, there is no `auth.revoke` call here: the - // caller already knows the bot token is dead (Slack told us via the - // event, or reconciliation just confirmed it via `auth.test`), so both - // secret columns are always dropped below. - if (row.apiKeyId) { - const keyId = decodeApiKeyIdOption(row.apiKeyId) - if (Option.isSome(keyId)) { - yield* apiKeys.revoke(orgId, keyId.value).pipe(Effect.ignore) - } - } - yield* database + // Compare-and-set on the snapshot: a concurrent `completeInstall` reuses + // the same row id and writes fresh secrets plus a newly minted API key, so + // an unconditional update-by-id would mark that fresh install revoked, null + // its secrets, and leave the new full-access key active but orphaned. The + // `updatedAt` guard makes the transition apply only to the exact version + // probed dead; a lost race leaves the reinstall alone (the hourly + // reconciliation re-probes it if its token is also dead). + const transitioned = yield* database .execute((db) => db .update(slackWorkspaces) @@ -1322,9 +1318,35 @@ const make: Effect.Effect< botTokenIv: null, botTokenTag: null, }) - .where(eq(slackWorkspaces.id, row.id)), + .where( + and( + eq(slackWorkspaces.id, row.id), + isNull(slackWorkspaces.revokedAt), + eq(slackWorkspaces.updatedAt, row.updatedAt), + ), + ) + .returning({ id: slackWorkspaces.id }), ) .pipe(Effect.mapError(toPersistenceError)) + if (transitioned.length === 0) { + yield* Effect.logInfo("Slack revocation lost a race to a concurrent write; leaving row as-is", { + orgId, + teamId, + reason, + }) + return { revoked: false } + } + // Revoke the minted API key only for the version we actually transitioned + // (best-effort — bookkeeping must not fail the revoke). Unlike `uninstall`, + // there is no `auth.revoke` call here: the caller already knows the bot + // token is dead (Slack told us via the event, or reconciliation just + // confirmed it via `auth.test`), so both secret columns were dropped above. + if (row.apiKeyId) { + const keyId = decodeApiKeyIdOption(row.apiKeyId) + if (Option.isSome(keyId)) { + yield* apiKeys.revoke(orgId, keyId.value).pipe(Effect.ignore) + } + } yield* Effect.logInfo("Slack workspace revoked remotely", { orgId, teamId, reason }) return { revoked: true } }) diff --git a/apps/api/src/services/integrations/slack-bot-token.test.ts b/apps/api/src/services/integrations/slack-bot-token.test.ts new file mode 100644 index 000000000..44c71c664 --- /dev/null +++ b/apps/api/src/services/integrations/slack-bot-token.test.ts @@ -0,0 +1,38 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { Effect } from "effect" +import { Database, DatabaseError, type DatabaseApi } from "@/platform/DatabaseLive" +import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" +import { resolveSlackBotTokenForDispatch } from "./slack-bot-token" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const ENCRYPTION_KEY = Buffer.alloc(32, 7) + +describe("resolveSlackBotTokenForDispatch", () => { + it.effect("classifies a database lookup failure as retryable, not as an auth failure", () => { + // A transient Postgres blip must re-enqueue the alert, not permanently drop + // it — AlertDeliveryAuthError is `retry: "never"` and counts toward + // auto-disabling the destination. + const failing: DatabaseApi = { + execute: () => Effect.fail(new DatabaseError({ message: "connection reset", cause: "boom" })), + } + return Effect.gen(function* () { + const error = yield* resolveSlackBotTokenForDispatch(failing, ENCRYPTION_KEY, "org_slack").pipe( + Effect.flip, + ) + assert.strictEqual(error._tag, "@maple/http/errors/AlertDeliveryError") + }) + }) + + it.effect("still classifies a missing installation as a terminal auth failure", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const database = yield* Database + const error = yield* resolveSlackBotTokenForDispatch(database, ENCRYPTION_KEY, "org_slack").pipe( + Effect.flip, + ) + assert.strictEqual(error._tag, "@maple/http/errors/AlertDeliveryAuthError") + }).pipe(Effect.provide(testDb.layer)) + }) +}) diff --git a/apps/api/src/services/integrations/slack-bot-token.ts b/apps/api/src/services/integrations/slack-bot-token.ts index 7d334eff0..ac1a4e703 100644 --- a/apps/api/src/services/integrations/slack-bot-token.ts +++ b/apps/api/src/services/integrations/slack-bot-token.ts @@ -1,5 +1,5 @@ import { slackWorkspaces } from "@maple/db" -import { AlertDeliveryAuthError, OrgId } from "@maple/domain/http" +import { AlertDeliveryAuthError, AlertDeliveryError, OrgId } from "@maple/domain/http" import { and, eq, isNull } from "drizzle-orm" import { Context, Effect, Layer, Option, Redacted, Schema } from "effect" import { decryptAes256Gcm, parseBase64Aes256GcmKey } from "@/platform/Crypto" @@ -59,8 +59,19 @@ export const resolveSlackBotTokenForDispatch = Effect.fn("SlackBotTokenResolver. ) { const decodedOrgId = yield* Schema.decodeEffect(OrgId)(orgId).pipe(Effect.orDie) yield* Effect.annotateCurrentSpan({ orgId: decodedOrgId }) + // A failed LOOKUP is not a failed authentication: the workspace may be + // connected and healthy behind a transient Postgres blip. Keep it retryable + // so the queue re-enqueues instead of dropping the alert (and so it does not + // count toward auto-disabling the destination). const rowOption = yield* loadActiveWorkspaceByOrg(database, decodedOrgId).pipe( - Effect.mapError((error) => notConnected(`Failed to load Slack installation: ${error.message}`)), + Effect.mapError( + (error) => + new AlertDeliveryError({ + message: "Failed to load the Slack installation", + destinationType: "slack-bot", + cause: error, + }), + ), ) if (Option.isNone(rowOption)) { return yield* Effect.fail( @@ -87,7 +98,7 @@ class SlackBotTokenConfigError extends Schema.TaggedError Effect.Effect + readonly resolve: (orgId: OrgId) => Effect.Effect } const make: Effect.Effect = Effect.gen( diff --git a/apps/api/src/services/integrations/vcs/VcsRepository.ts b/apps/api/src/services/integrations/vcs/VcsRepository.ts index f3474157a..d9494488e 100644 --- a/apps/api/src/services/integrations/vcs/VcsRepository.ts +++ b/apps/api/src/services/integrations/vcs/VcsRepository.ts @@ -441,12 +441,21 @@ export class VcsRepository extends Context.Service()("@maple/api/ createdAt: now, updatedAt: now, })) - yield* Effect.forEach( - Arr.chunksOf(values, INSERT_CHUNK_SIZE), - (chunk) => - database - .execute((db) => - db + // Share-lock the owning installation for the duration of the write. A purge + // deletes the installation row FIRST, so it either waits for this lock and + // then sweeps these rows, or has already won — and a worker holding a stale + // snapshot writes nothing instead of resurrecting purged data. + yield* database + .execute((db) => + db.transaction(async (tx) => { + const parent = await tx + .select({ id: vcsInstallations.id }) + .from(vcsInstallations) + .where(eq(vcsInstallations.id, installation.id)) + .for("share") + if (parent.length === 0) return + for (const chunk of Arr.chunksOf(values, INSERT_CHUNK_SIZE)) { + await tx .insert(vcsRepositories) .values(chunk) .onConflictDoUpdate({ @@ -471,11 +480,11 @@ export class VcsRepository extends Context.Service()("@maple/api/ status: sql`excluded.status`, updatedAt: sql`excluded.updated_at`, }, - }), - ) - .pipe(Effect.mapError(toPersistenceError)), - { discard: true }, - ) + }) + } + }), + ) + .pipe(Effect.mapError(toPersistenceError)) }) // Soft-delete: the provider revoked access to this repo. The row and its @@ -498,8 +507,7 @@ export class VcsRepository extends Context.Service()("@maple/api/ // Hard-delete a single repo and its commits by Maple's own repository id. // User-initiated only: the dashboard "delete from Maple" action. Confirm the // row exists for this org first (so a foreign/absent id is a no-op, not a - // blind delete), then delete commits — which reference the repo by this id — - // before the row, so a mid-failure can't orphan them. Idempotent. + // blind delete). Idempotent. const purgeRepository = Effect.fn("VcsRepository.purgeRepository")(function* ( orgId: OrgId, repositoryId: VcsRepositoryId, @@ -512,15 +520,17 @@ export class VcsRepository extends Context.Service()("@maple/api/ .limit(1), ) if (repoRows[0]?.id === undefined) return false - // Delete branches, commits, then the repo in one atomic transaction, so a - // failure part-way can't leave child rows behind a deleted repo. + // One atomic transaction, PARENT FIRST: deleting the repo row before its + // children pairs with the share-lock gates in the child upserts — an + // in-flight sync either commits before this delete acquires the row lock + // (its rows are swept below) or sees the row gone and writes nothing. yield* database.execute((db) => db.transaction(async (tx) => { + await tx.delete(vcsRepositories).where(eq(vcsRepositories.id, repositoryId)) await tx .delete(vcsRepositoryBranches) .where(eq(vcsRepositoryBranches.repositoryId, repositoryId)) await tx.delete(vcsCommits).where(eq(vcsCommits.repositoryId, repositoryId)) - await tx.delete(vcsRepositories).where(eq(vcsRepositories.id, repositoryId)) }), ) return true @@ -611,12 +621,20 @@ export class VcsRepository extends Context.Service()("@maple/api/ }) // Upsert the immutable commit rows, refreshing mutable metadata on conflict. - yield* Effect.forEach( - Arr.chunksOf(values, INSERT_CHUNK_SIZE), - (chunk) => - database - .execute((db) => - db + // Gated on the share-locked repo row (see upsertRepositories): a worker that + // waited on GitHub across a purge must not reinsert a deleted repo's private + // commits. Returns 0 when the repo is gone. + return yield* database + .execute((db) => + db.transaction(async (tx) => { + const parent = await tx + .select({ id: vcsRepositories.id }) + .from(vcsRepositories) + .where(eq(vcsRepositories.id, repository.id)) + .for("share") + if (parent.length === 0) return 0 + for (const chunk of Arr.chunksOf(values, INSERT_CHUNK_SIZE)) { + await tx .insert(vcsCommits) .values(chunk) .onConflictDoUpdate({ @@ -631,30 +649,33 @@ export class VcsRepository extends Context.Service()("@maple/api/ committedAt: sql`excluded.committed_at`, htmlUrl: sql`excluded.html_url`, }, - }), - ) - .pipe(Effect.mapError(toPersistenceError)), - { discard: true }, - ) - return values.length + }) + } + return values.length + }), + ) + .pipe(Effect.mapError(toPersistenceError)) }) const findCommitBySha = Effect.fn("VcsRepository.findCommitBySha")(function* ( orgId: OrgId, sha: GitCommitSha, ) { + // Joins the owning repo (existence only — no columns read) so a commit + // orphaned by a purge racing a stale write is unreadable, not leaked. const rows = yield* database .execute((db) => db - .select() + .select({ commit: vcsCommits }) .from(vcsCommits) + .innerJoin(vcsRepositories, eq(vcsCommits.repositoryId, vcsRepositories.id)) .where(and(eq(vcsCommits.orgId, orgId), eq(vcsCommits.sha, sha))) .limit(1), ) .pipe(Effect.mapError(toPersistenceError)) const row = Option.fromNullishOr(rows[0]) if (Option.isNone(row)) return Option.none() - return Option.some(yield* decodeOne("vcs_commits", row.value, rowToCommit)) + return Option.some(yield* decodeOne("vcs_commits", row.value.commit, rowToCommit)) }) // Bulk sibling of findCommitBySha, for resolving a whole table of deploy @@ -668,12 +689,14 @@ export class VcsRepository extends Context.Service()("@maple/api/ const rows = yield* database .execute((db) => db - .select() + .select({ commit: vcsCommits }) .from(vcsCommits) + // Same orphan shield as findCommitBySha. + .innerJoin(vcsRepositories, eq(vcsCommits.repositoryId, vcsRepositories.id)) .where(and(eq(vcsCommits.orgId, orgId), inArray(vcsCommits.sha, [...new Set(shas)]))), ) .pipe(Effect.mapError(toPersistenceError)) - return yield* Effect.forEach(rows, (row) => decodeOne("vcs_commits", row, rowToCommit)) + return yield* Effect.forEach(rows, (row) => decodeOne("vcs_commits", row.commit, rowToCommit)) }) // Bulk upsert a repo's branches from a provider listing — just the picker's @@ -709,12 +732,19 @@ export class VcsRepository extends Context.Service()("@maple/api/ column: "head_sha", }), }) - yield* Effect.forEach( - Arr.chunksOf(values, INSERT_CHUNK_SIZE), - (chunk) => - database - .execute((db) => - db + // Gated on the share-locked repo row (see upsertRepositories) so a stale + // worker cannot repopulate the picker of a purged repo. + yield* database + .execute((db) => + db.transaction(async (tx) => { + const parent = await tx + .select({ id: vcsRepositories.id }) + .from(vcsRepositories) + .where(eq(vcsRepositories.id, repository.id)) + .for("share") + if (parent.length === 0) return + for (const chunk of Arr.chunksOf(values, INSERT_CHUNK_SIZE)) { + await tx .insert(vcsRepositoryBranches) .values(chunk) .onConflictDoUpdate({ @@ -724,11 +754,11 @@ export class VcsRepository extends Context.Service()("@maple/api/ headSha: sql`excluded.head_sha`, updatedAt: sql`excluded.updated_at`, }, - }), - ) - .pipe(Effect.mapError(toPersistenceError)), - { discard: true }, - ) + }) + } + }), + ) + .pipe(Effect.mapError(toPersistenceError)) }) // Resolve a branch by name, creating it if absent (a push can surface a branch @@ -889,55 +919,52 @@ export class VcsRepository extends Context.Service()("@maple/api/ }) // Remove an installation and everything beneath it (its repositories and - // their commits), in dependency order. The dashboard disconnect flow uses - // this: severing the integration must not strand the org's repos/commits in - // the VCS tables. Idempotent — re-running drops whatever still remains. - // - // Commits reference their repo by internal id, so the installation's repo - // ids are resolved first and used to delete the commits; the repo and - // installation rows are then deleted in the SAME atomic batch. + // their commits). The dashboard disconnect flow uses this: severing the + // integration must not strand the org's repos/commits in the VCS tables. + // Idempotent — re-running drops whatever still remains. const purgeInstallation = Effect.fn("VcsRepository.purgeInstallation")(function* ( orgId: OrgId, installationId: VcsInstallationId, ) { - const repoRows = yield* database.execute((db) => - db - .select({ id: vcsRepositories.id }) - .from(vcsRepositories) - .where( - and( - eq(vcsRepositories.orgId, orgId), - eq(vcsRepositories.installationId, installationId), - ), - ), - ) - - const repoIds = repoRows.map((r) => r.id) - // One atomic transaction, children before parents: branches + commits (chunked - // to stay under the bind-variable cap), then the repos, then the installation. + // One atomic transaction, PARENTS FIRST: the installation, then its repos, + // then their branches + commits (chunked under the bind-variable cap). With + // the share-lock gates in the upserts, an in-flight sync either commits + // before a parent delete (its rows are swept below) or writes nothing. The + // repo-id read happens INSIDE the transaction, after the installation + // delete, so a repo created moments earlier is swept rather than escaping. yield* database.execute((db) => db.transaction(async (tx) => { - for (const chunk of Arr.chunksOf(repoIds, INARRAY_CHUNK_SIZE)) { - await tx - .delete(vcsRepositoryBranches) - .where(inArray(vcsRepositoryBranches.repositoryId, chunk)) - } - for (const chunk of Arr.chunksOf(repoIds, INARRAY_CHUNK_SIZE)) { - await tx.delete(vcsCommits).where(inArray(vcsCommits.repositoryId, chunk)) - } await tx - .delete(vcsRepositories) + .delete(vcsInstallations) + .where( + and(eq(vcsInstallations.orgId, orgId), eq(vcsInstallations.id, installationId)), + ) + const repoRows = await tx + .select({ id: vcsRepositories.id }) + .from(vcsRepositories) .where( and( eq(vcsRepositories.orgId, orgId), eq(vcsRepositories.installationId, installationId), ), ) + const repoIds = repoRows.map((r) => r.id) await tx - .delete(vcsInstallations) + .delete(vcsRepositories) .where( - and(eq(vcsInstallations.orgId, orgId), eq(vcsInstallations.id, installationId)), + and( + eq(vcsRepositories.orgId, orgId), + eq(vcsRepositories.installationId, installationId), + ), ) + for (const chunk of Arr.chunksOf(repoIds, INARRAY_CHUNK_SIZE)) { + await tx + .delete(vcsRepositoryBranches) + .where(inArray(vcsRepositoryBranches.repositoryId, chunk)) + } + for (const chunk of Arr.chunksOf(repoIds, INARRAY_CHUNK_SIZE)) { + await tx.delete(vcsCommits).where(inArray(vcsCommits.repositoryId, chunk)) + } }), ) }, Effect.mapError(toPersistenceError)) diff --git a/apps/api/src/services/integrations/vcs/VcsSyncService.ts b/apps/api/src/services/integrations/vcs/VcsSyncService.ts index 3c9db17d5..af3320ebc 100644 --- a/apps/api/src/services/integrations/vcs/VcsSyncService.ts +++ b/apps/api/src/services/integrations/vcs/VcsSyncService.ts @@ -156,6 +156,28 @@ export class VcsSyncService extends Context.Service Effect.gen(function* () { + // A tracked-branch change wipes the repo's commits and enqueues a fresh + // backfill, but old-branch jobs survive in the queue (a rate-limited + // continuation can sit delayed for hours). Drop them — and re-check + // after the provider fetch below — so a stale job can never repopulate + // the wiped set, mark it ready, or overwrite the new backfill's status. + if (job.branch !== trackedBranchOf(repository)) { + yield* Effect.annotateCurrentSpan({ + "vcs.commits.outcome": "skipped", + "vcs.commits.reason": "stale_tracked_branch", + }) + yield* Effect.logInfo( + "[VCS] Dropping commit sync for a branch no longer tracked", + ).pipe( + Effect.annotateLogs({ + provider: installation.provider, + externalRepoId: job.externalRepoId, + branch: job.branch, + trackedBranch: trackedBranchOf(repository), + }), + ) + return + } // Mark the backfill in progress before the first provider call — the // execution path owns every sync_status transition, so this is what the // dashboard sees the moment a (re)sync actually starts (e.g. after a @@ -186,6 +208,19 @@ export class VcsSyncService extends Context.Service other.provider === active.provider && other.id !== active.id, + (other) => + other.provider === active.provider && + other.id !== active.id && + (other.createdAt < active.createdAt || + (other.createdAt === active.createdAt && other.id < active.id)), ) if (superseded.length > 0) { yield* Effect.forEach( @@ -940,6 +985,15 @@ export class VcsSyncService extends Context.Service { }, ) + // Regression: a queue worker resolved its installation/repo, waited on GitHub, + // and raced a purge — its late writes used to recreate rows under a deleted + // parent, and a purged private commit stayed queryable by (org, sha). With no + // FK (house style), the repo layer must make the stale write a silent no-op + // and shield the (org, sha) reads from any orphan. + it.effect("stale snapshots cannot resurrect purged data, and orphans are unreadable", () => { + const testDb = createTestDb(trackedDbs) + const SHA = "c".repeat(40) + const commitFixture = { + sha: SHA, + message: "m", + authorName: null, + authorEmail: null, + authorLogin: null, + authorAvatarUrl: null, + authoredAt: null, + committedAt: 1, + htmlUrl: `https://github.com/octo/repo/commit/${SHA}`, + branch: "main", + } + const countRows = (table: string) => + Effect.promise(() => + queryFirstRow<{ n: number }>(testDb, `SELECT count(*)::int AS n FROM ${table}`), + ).pipe(Effect.map((row) => row?.n ?? 0)) + return Effect.gen(function* () { + const repo = yield* VcsRepository + const orgId = asOrgId("org_fk") + const installation = yield* repo.upsertInstallation({ orgId, ...installationSeed("42", "100") }) + yield* repo.upsertRepositories(installation, [repoFixture()]) + const r = yield* repoFor(repo, orgId, "7") + assert.strictEqual(yield* repo.upsertCommits(r, [commitFixture]), 1) + + // Purge the repo, then replay the writes of a worker still holding `r`: + // every one must be a no-op, not a resurrection. + assert.ok(yield* repo.purgeRepository(orgId, r.id)) + assert.strictEqual(yield* repo.upsertCommits(r, [commitFixture]), 0) + yield* repo.upsertBranches(r, [{ name: "main", headSha: null }]) + assert.strictEqual(yield* countRows("vcs_commits"), 0) + assert.strictEqual(yield* countRows("vcs_repository_branches"), 0) + assert.ok(Option.isNone(yield* repo.findCommitBySha(orgId, decodeGitCommitSha(SHA)))) + + // Purge the installation, then replay a repo upsert from a stale snapshot. + yield* repo.purgeInstallation(orgId, installation.id) + yield* repo.upsertRepositories(installation, [repoFixture()]) + assert.strictEqual(yield* countRows("vcs_repositories"), 0) + assert.ok(Option.isNone(yield* repo.resolveRepository(orgId, "github", "7"))) + + // Read shield: even a manufactured orphan (no parent repo row) must not + // surface through the (org, sha) lookups. The shield is application-level: + // there is deliberately no FK, so the row inserts and only the join hides it. + yield* Effect.promise(() => + executeSql( + testDb, + `INSERT INTO vcs_commits + (id, org_id, provider, repository_id, sha, message, html_url, committed_at, created_at) + VALUES ($1,$2,$3,$4,$5,$6,$7, now(), now())`, + [randomUUID(), orgId, "github", randomUUID(), SHA, "m", "https://example.com"], + ), + ) + assert.ok(Option.isNone(yield* repo.findCommitBySha(orgId, decodeGitCommitSha(SHA)))) + assert.strictEqual((yield* repo.findCommitsByShas(orgId, [decodeGitCommitSha(SHA)])).length, 0) + }).pipe(Effect.provide(repoLayer(testDb))) + }) + const installationSeed = (externalInstallationId: string, externalAccountId: string) => ({ provider: "github" as const, externalInstallationId, @@ -1332,6 +1396,12 @@ describe("VcsSyncService orchestrator", () => { isArchived: boolean }> readonly commits?: ReadonlyArray> + /** + * Runs inside the stubbed provider's fetchCommits, before it answers — + * the hook point for simulating a concurrent write (e.g. a tracked-branch + * retarget) landing while the sync is mid-flight at the provider. + */ + readonly onFetchCommits?: () => Promise readonly commitFetchNext?: { untilMs: number retryAfterSeconds: number @@ -1359,12 +1429,16 @@ describe("VcsSyncService orchestrator", () => { fetchRepositories: () => opts.fetchReposError ? Effect.fail(opts.fetchReposError) : Effect.succeed(opts.repos ?? []), fetchCommits: () => - opts.fetchCommitsError - ? Effect.fail(opts.fetchCommitsError) - : Effect.succeed({ - commits: opts.commits ?? [], - ...(opts.commitFetchNext ? { next: opts.commitFetchNext } : undefined), - }), + Effect.promise(async () => opts.onFetchCommits?.()).pipe( + Effect.andThen( + opts.fetchCommitsError + ? Effect.fail(opts.fetchCommitsError) + : Effect.succeed({ + commits: opts.commits ?? [], + ...(opts.commitFetchNext ? { next: opts.commitFetchNext } : undefined), + }), + ), + ), fetchBranches: () => opts.fetchBranchesError ? Effect.fail(opts.fetchBranchesError) @@ -1961,6 +2035,15 @@ describe("VcsSyncService orchestrator", () => { const STALE_SHA = "c".repeat(40) yield* upsertCommitsFor(repo, orgId, "70", [commit(STALE_SHA, 1)]) + // The stale row predates the new install in reality; backdate it so the + // strict-order purge (strictly-older siblings only) applies even when both + // test seeds land in the same clock millisecond. + yield* Effect.promise(() => + executeSql( + testDb, + `UPDATE vcs_installations SET created_at = created_at - interval '1 hour' WHERE external_installation_id = '11'`, + ), + ) yield* seedInstallation(repo, orgId) const job: VcsSyncJob = { kind: "installation-sync", @@ -2914,6 +2997,126 @@ describe("VcsSyncService orchestrator", () => { assert.strictEqual(sent.length, 0) }).pipe(Effect.provide(orchestratorLayer(testDb, { sent }))) }) + + // Regression: a sync-commits job enqueued for the previously tracked branch + // (queued, delayed, or a continuation) must never repopulate the wiped commit + // set or flip the new backfill's status. + it.effect("drops a sync-commits job whose branch is no longer tracked", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* seedRepo(repo) + const r = yield* repoFor(repo, orgId, "7") + yield* repo.changeTrackedBranch(orgId, r.id, "release") + // The queued job still names the old tracked branch ("main"). + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(backfillJob)) + assert.ok(Option.isNone(yield* repo.findCommitBySha(orgId, decodeGitCommitSha(SHA_A)))) + const stored = yield* reposOfInstallation(repo, "42", "all") + const head = Option.getOrThrow(Arr.head(stored)) + assert.strictEqual(head.syncStatus, "pending") // untouched — not "ready" + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent, commits: [commit(SHA_A, 1)] }))) + }) + + it.effect("a retarget landing during the provider fetch invalidates the walk before it writes", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* seedRepo(repo) + // The retarget (simulated inside the provider fetch below) wins the race: + // the stale walk must not write its old-branch commits or mark it ready. + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(backfillJob)) + assert.ok(Option.isNone(yield* repo.findCommitBySha(orgId, decodeGitCommitSha(SHA_A)))) + const stored = yield* reposOfInstallation(repo, "42", "all") + const head = Option.getOrThrow(Arr.head(stored)) + assert.notStrictEqual(head.syncStatus, "ready") + }).pipe( + Effect.provide( + orchestratorLayer(testDb, { + sent, + commits: [commit(SHA_A, 1)], + onFetchCommits: () => + executeSql( + testDb, + `UPDATE vcs_repositories SET tracked_branch = 'release' WHERE external_repo_id = '7'`, + ), + }), + ), + ) + }) + + it.effect("an exhausted stale-branch job does not mark the new branch's backfill errored", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) + yield* seedRepo(repo) + const r = yield* repoFor(repo, orgId, "7") + yield* repo.changeTrackedBranch(orgId, r.id, "release") + // The exhausted job walked the OLD branch; the new backfill is untouched. + yield* svc.recordExhaustedFailure(Schema.encodeSync(VcsSyncJob)(backfillJob)) + const stored = yield* reposOfInstallation(repo, "42", "all") + const head = Option.getOrThrow(Arr.head(stored)) + assert.strictEqual(head.syncStatus, "pending") + assert.strictEqual(head.lastSyncError, null) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent }))) + }) + + // Regression: two near-simultaneous created/updated jobs for different + // installations used to each see the other as a sibling and mutually purge + // both. "Supersedes" is now a strict order — only strictly-older siblings are + // purged — so the newest installation deterministically survives. + it.effect("a created job never purges a newer sibling installation", () => { + const testDb = createTestDb(trackedDbs) + const sent: Array = [] + return Effect.gen(function* () { + const svc = yield* VcsSyncService + const repo = yield* VcsRepository + const orgId = asOrgId("org_orch") + yield* seedInstallation(repo, orgId) // external id "42" + yield* repo.upsertInstallation({ + orgId, + provider: "github", + externalInstallationId: "43", + accountLogin: "octo2", + accountType: "organization", + externalAccountId: "101", + accountAvatarUrl: null, + repositorySelection: "all", + installedByUserId: asUserId("user_1"), + }) + // Make "42" strictly older so the winner is deterministic. + yield* Effect.promise(() => + executeSql( + testDb, + `UPDATE vcs_installations SET created_at = created_at - interval '1 hour' WHERE external_installation_id = '42'`, + ), + ) + const createdJob = (id: string): VcsSyncJob => ({ + kind: "installation-sync", + provider: "github", + externalInstallationId: id, + reason: "created", + }) + // The OLDER installation's job must NOT purge the newer sibling. + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(createdJob("42"))) + assert.ok(Option.isSome(yield* repo.resolveInstallation("github", "43"))) + // The NEWER installation's job supersedes and purges the older. + yield* svc.processMessage(Schema.encodeSync(VcsSyncJob)(createdJob("43"))) + assert.ok(Option.isNone(yield* repo.resolveInstallation("github", "42"))) + assert.ok(Option.isSome(yield* repo.resolveInstallation("github", "43"))) + }).pipe(Effect.provide(orchestratorLayer(testDb, { sent }))) + }) }) // The SHA-shape regex lives only in the GitCommitSha brand; these assert that diff --git a/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts b/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts index 89cc475c6..3b2caa43d 100644 --- a/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts +++ b/apps/api/src/services/org/OrgClickHouseSettingsService.test.ts @@ -8,6 +8,7 @@ import { OrgClickHouseSettingsValidationError, OrgId, RoleName, + UserId, } from "@maple/domain/http" import { EdgeCacheService, @@ -21,9 +22,11 @@ import { FetchHttpClient } from "effect/unstable/http" import type { TableDiffEntry } from "@maple/domain/clickhouse" import { Env } from "@/platform/Env" import { encryptAes256Gcm } from "@/platform/Crypto" -import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@/platform/test-pglite" +import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { cleanupTestDbs, createTestDb, executeSql, queryFirstRow, type TestDb } from "@/platform/test-pglite" import { type ClickHouseExecConfig, + decodeSkippedEntries, execClickHouse, invalidateOrgRuntimeConfigMemo, isRetryableUpstream, @@ -140,6 +143,30 @@ describe("shouldHealSchemaVersion", () => { }) }) +describe("decodeSkippedEntries", () => { + it("surfaces the workflow's skipped-migration entries from the run row", () => { + // A non-gating migration that fails after its DROP VIEW leaves the view + // absent while the run still reports "succeeded" — the skipped list is the + // only operator-visible trace, so the status endpoint must carry it. + expect( + decodeSkippedEntries([ + { id: "migration_20", reason: "ClickHouse 241: quota exceeded" }, + { id: "feature_reconciliation", reason: "version probe failed" }, + ]), + ).toEqual([ + { id: "migration_20", reason: "ClickHouse 241: quota exceeded" }, + { id: "feature_reconciliation", reason: "version probe failed" }, + ]) + }) + + it("reads unrecognised or absent jsonb shapes as no skips", () => { + expect(decodeSkippedEntries(null)).toEqual([]) + expect(decodeSkippedEntries(undefined)).toEqual([]) + expect(decodeSkippedEntries("oops")).toEqual([]) + expect(decodeSkippedEntries([{ name: "legacy-shape", reason: "x" }])).toEqual([]) + }) +}) + describe("isRetryableUpstream", () => { it("retries transient gateway/proxy codes and network failures, nothing else", () => { // Transient → retry. @@ -874,3 +901,149 @@ describe("resolveRuntimeConfig caching", () => { }).pipe(Effect.provide(buildLayer(testDb))) }) }) + +describe("applySchema claim lifecycle", () => { + const applyTrackedDbs: TestDb[] = [] + afterEach(() => cleanupTestDbs(applyTrackedDbs)) + + const asOrgId = Schema.decodeUnknownSync(OrgId) + const asRole = Schema.decodeUnknownSync(RoleName) + const asUserIdApply = Schema.decodeUnknownSync(UserId) + const ADMIN = [asRole("org:admin")] + + const applyConfigLive = ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://maple-managed.tinybird.co", + TINYBIRD_TOKEN: "managed-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 5).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "lookup-key", + MAPLE_INGEST_PUBLIC_URL: "http://127.0.0.1:3474", + MAPLE_APP_BASE_URL: "http://127.0.0.1:3471", + }), + ) + + const missBackend: EdgeCacheBackend = { + name: "memory", + get: () => Promise.resolve(undefined), + put: () => Promise.resolve(), + delete: () => Promise.resolve(), + } + + const buildApplyLayer = (testDb: TestDb, workerEnv: Record) => { + const envLive = Env.layer.pipe(Layer.provide(applyConfigLive)) + const edgeCacheLive = Layer.succeed(EdgeCacheService)(makeEdgeCacheService(missBackend)) + return OrgClickHouseSettingsService.layer.pipe( + Layer.provide( + Layer.mergeAll( + envLive, + testDb.layer, + edgeCacheLive, + Layer.succeed(WorkerEnvironment)(workerEnv), + ), + ), + ) + } + + const seedSettings = (testDb: TestDb, orgId: string) => + executeSql( + testDb, + `INSERT INTO org_clickhouse_settings + (org_id, ch_url, ch_user, ch_database, sync_status, created_at, updated_at, created_by, updated_by) + VALUES ($1, 'https://clickhouse.example.test', 'default', 'maple', 'connected', NOW(), NOW(), 'u', 'u')`, + [orgId], + ) + + const runStatus = (testDb: TestDb, orgId: string) => + queryFirstRow<{ status: string }>( + testDb, + "SELECT status FROM org_clickhouse_schema_apply_runs WHERE org_id = $1", + [orgId], + ) + + it.effect("a workflow-creation failure releases the claim instead of wedging on already_running", () => { + const testDb = createTestDb(applyTrackedDbs) + const orgId = "org_apply_create_fails" + let attempts = 0 + const binding = { + create: () => { + attempts += 1 + return attempts === 1 + ? Promise.reject(new Error("workflow backend unavailable")) + : Promise.resolve({ id: "wf-1" }) + }, + } + return Effect.gen(function* () { + yield* Effect.promise(() => seedSettings(testDb, orgId)) + const service = yield* OrgClickHouseSettingsService + + const first = yield* service + .applySchema(asOrgId(orgId), asUserIdApply("user_a"), ADMIN) + .pipe(Effect.exit) + expect(Exit.isFailure(first)).toBe(true) + // The claim was released — failed, not queued. + expect((yield* Effect.promise(() => runStatus(testDb, orgId)))?.status).toBe("failed") + + // A retry is possible without manual database repair. + const second = yield* service.applySchema(asOrgId(orgId), asUserIdApply("user_a"), ADMIN) + expect(second.status).toBe("started") + expect(attempts).toBe(2) + }).pipe(Effect.provide(buildApplyLayer(testDb, { CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: binding }))) + }) + + it.effect("a missing workflow binding fails before any claim is written", () => { + const testDb = createTestDb(applyTrackedDbs) + const orgId = "org_apply_no_binding" + return Effect.gen(function* () { + yield* Effect.promise(() => seedSettings(testDb, orgId)) + const service = yield* OrgClickHouseSettingsService + const exit = yield* service + .applySchema(asOrgId(orgId), asUserIdApply("user_a"), ADMIN) + .pipe(Effect.exit) + expect(Exit.isFailure(exit)).toBe(true) + // No leftover queued row — the next attempt starts from idle. + expect(yield* Effect.promise(() => runStatus(testDb, orgId))).toBeUndefined() + }).pipe(Effect.provide(buildApplyLayer(testDb, {}))) + }) + + it.effect("an active claim blocks a second start, and a stale one is reclaimed", () => { + const testDb = createTestDb(applyTrackedDbs) + const orgId = "org_apply_stale" + let creates = 0 + const binding = { + create: () => { + creates += 1 + return Promise.resolve({ id: `wf-${creates}` }) + }, + } + return Effect.gen(function* () { + yield* Effect.promise(() => seedSettings(testDb, orgId)) + const service = yield* OrgClickHouseSettingsService + + const first = yield* service.applySchema(asOrgId(orgId), asUserIdApply("user_a"), ADMIN) + expect(first.status).toBe("started") + + // The row is queued and fresh: the claim is exclusive. + const second = yield* service.applySchema(asOrgId(orgId), asUserIdApply("user_a"), ADMIN) + expect(second.status).toBe("already_running") + expect(creates).toBe(1) + + // A queued row abandoned for over 30 minutes (workflow died without + // reaching its catch) must not block schema application forever. + yield* Effect.promise(() => + executeSql( + testDb, + // Relative to the TestClock (which sits at the epoch), not wall time. + "UPDATE org_clickhouse_schema_apply_runs SET updated_at = to_timestamp(0) - interval '31 minutes' WHERE org_id = $1", + [orgId], + ), + ) + const third = yield* service.applySchema(asOrgId(orgId), asUserIdApply("user_a"), ADMIN) + expect(third.status).toBe("started") + expect(creates).toBe(2) + }).pipe(Effect.provide(buildApplyLayer(testDb, { CLICKHOUSE_SCHEMA_APPLY_WORKFLOW: binding }))) + }) +}) diff --git a/apps/api/src/services/org/OrgClickHouseSettingsService.ts b/apps/api/src/services/org/OrgClickHouseSettingsService.ts index 3fcbd4e57..ea0f1db22 100644 --- a/apps/api/src/services/org/OrgClickHouseSettingsService.ts +++ b/apps/api/src/services/org/OrgClickHouseSettingsService.ts @@ -32,7 +32,7 @@ import { } from "@maple/domain/clickhouse" import { EdgeCacheService } from "@maple/cache" import { orgClickHouseSchemaApplyRuns, orgClickHouseSettings } from "@maple/db" -import { eq, inArray } from "drizzle-orm" +import { and, eq, inArray, lt, notInArray, or } from "drizzle-orm" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" import { Array as Arr, @@ -263,6 +263,22 @@ const ROOT_ROLE = Schema.decodeSync(RoleName)("root") const ORG_ADMIN_ROLE = Schema.decodeSync(RoleName)("org:admin") const decodeIsoDateTimeStringSync = Schema.decodeUnknownSync(IsoDateTimeString) +const SkippedEntries = Schema.Array(Schema.Struct({ id: Schema.String, reason: Schema.String })) +const decodeSkippedEntriesOption = Schema.decodeUnknownOption(SkippedEntries) + +/** + * The run row's `skipped` jsonb, written by the schema-apply workflow as + * `{ id, reason }` entries (non-gating migrations and optional features it + * could not apply). Exposed on the status response: a "succeeded" run that + * silently skipped a failed view recreation left the cluster without that + * writer, and nothing else tells the operator to re-apply. Lenient — an + * unrecognised shape (old rows, nulls) reads as no skips. + */ +export const decodeSkippedEntries = ( + value: unknown, +): ReadonlyArray<{ readonly id: string; readonly reason: string }> => + Option.getOrElse(decodeSkippedEntriesOption(value), () => []) + export interface OrgClickHouseSettingsServiceApi { readonly get: ( orgId: OrgId, @@ -391,6 +407,14 @@ const toPersistenceError = (error: unknown) => // schema apply. Resolved off the worker env at runtime — see `apply-schema`. const SCHEMA_APPLY_WORKFLOW_BINDING = "CLICKHOUSE_SCHEMA_APPLY_WORKFLOW" +/** + * A queued/running apply-run row whose `updatedAt` is older than this is + * treated as abandoned and may be reclaimed by a new applySchema call. The + * workflow touches the row on every durable step, so half an hour of silence + * means the instance died somewhere its catch could not reach. + */ +const STALE_APPLY_RUN_MS = 30 * 60_000 + interface WorkflowBinding { readonly create: (options?: { readonly id?: string @@ -1283,22 +1307,28 @@ export class OrgClickHouseSettingsService extends Context.Service< // Ensure BYO ClickHouse is configured before queuing a run. yield* requireActiveRow(orgId) - const existing = yield* database - .execute((db) => - db - .select() - .from(orgClickHouseSchemaApplyRuns) - .where(eq(orgClickHouseSchemaApplyRuns.orgId, orgId)) - .limit(1), + // Resolve the binding BEFORE claiming: a missing binding must not leave + // a queued row behind that every later attempt reads as already_running. + const binding = Option.match(workerEnv, { + onNone: () => undefined, + onSome: (e) => e[SCHEMA_APPLY_WORKFLOW_BINDING], + }) + if (!isWorkflowBinding(binding)) { + return yield* Effect.fail( + new OrgClickHouseSettingsPersistenceError({ + message: `Schema-apply workflow binding (${SCHEMA_APPLY_WORKFLOW_BINDING}) unavailable`, + }), ) - .pipe(Effect.mapError(toPersistenceError)) - const current = existing[0] - if (current && (current.status === "queued" || current.status === "running")) { - return new OrgClickHouseApplySchemaStarted({ status: "already_running" }) } + // Atomic claim: the conflict-update is gated so exactly one of two + // concurrent applySchema calls wins (interleaved workflow instances + // would race destructive DROP/TRUNCATE/backfill migrations). A + // queued/running row that stopped updating for STALE_APPLY_RUN_MS is + // reclaimable — the workflow stamps progress on every step, so a silent + // stall that long means the instance died without reaching its catch. const now = yield* Clock.currentTimeMillis - yield* database + const claimed = yield* database .execute((db) => db .insert(orgClickHouseSchemaApplyRuns) @@ -1320,6 +1350,13 @@ export class OrgClickHouseSettingsService extends Context.Service< }) .onConflictDoUpdate({ target: orgClickHouseSchemaApplyRuns.orgId, + setWhere: or( + notInArray(orgClickHouseSchemaApplyRuns.status, ["queued", "running"]), + lt( + orgClickHouseSchemaApplyRuns.updatedAt, + new Date(now - STALE_APPLY_RUN_MS), + ), + ), set: { status: "queued", phase: "queued", @@ -1333,28 +1370,45 @@ export class OrgClickHouseSettingsService extends Context.Service< finishedAt: null, updatedAt: new Date(now), }, - }), + }) + .returning({ orgId: orgClickHouseSchemaApplyRuns.orgId }), ) .pipe(Effect.mapError(toPersistenceError)) - - const binding = Option.match(workerEnv, { - onNone: () => undefined, - onSome: (e) => e[SCHEMA_APPLY_WORKFLOW_BINDING], - }) - if (!isWorkflowBinding(binding)) { - return yield* Effect.fail( - new OrgClickHouseSettingsPersistenceError({ - message: `Schema-apply workflow binding (${SCHEMA_APPLY_WORKFLOW_BINDING}) unavailable`, - }), - ) + if (claimed.length === 0) { + return new OrgClickHouseApplySchemaStarted({ status: "already_running" }) } + yield* Effect.tryPromise({ try: () => binding.create({ params: { orgId } }), catch: (error) => new OrgClickHouseSettingsPersistenceError({ message: `Failed to start schema-apply workflow: ${error instanceof Error ? error.message : String(error)}`, }), - }) + }).pipe( + // No workflow exists to move the claim off "queued", so release it + // here (best-effort) — otherwise the org is wedged on already_running + // until manual database repair. + Effect.tapError((error) => + database + .execute((db) => + db + .update(orgClickHouseSchemaApplyRuns) + .set({ + status: "failed", + errorMessage: error.message, + finishedAt: new Date(now), + updatedAt: new Date(now), + }) + .where( + and( + eq(orgClickHouseSchemaApplyRuns.orgId, orgId), + eq(orgClickHouseSchemaApplyRuns.status, "queued"), + ), + ), + ) + .pipe(Effect.ignore), + ), + ) return new OrgClickHouseApplySchemaStarted({ status: "started" }) }) @@ -1383,6 +1437,7 @@ export class OrgClickHouseSettingsService extends Context.Service< stepsTotal: null, stepsDone: null, appliedVersions: [], + skipped: [], errorMessage: null, startedAt: null, finishedAt: null, @@ -1406,6 +1461,7 @@ export class OrgClickHouseSettingsService extends Context.Service< stepsTotal: row.stepsTotal ?? null, stepsDone: row.stepsDone ?? null, appliedVersions, + skipped: decodeSkippedEntries(row.skipped), errorMessage: row.errorMessage ?? null, startedAt: dateToMs(row.startedAt), finishedAt: dateToMs(row.finishedAt), diff --git a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.test.ts b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.test.ts index d38365296..ac4ea4ae5 100644 --- a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.test.ts +++ b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.test.ts @@ -1,5 +1,13 @@ -import { describe, expect, it } from "@effect/vitest" -import { loadOptionalFeatureState } from "./ClickHouseSchemaApplyWorkflow.run" +import { afterEach, describe, expect, it } from "@effect/vitest" +import { Effect, Option } from "effect" +import { Database } from "@/platform/DatabaseLive" +import type { PgConnectionScopeApi } from "@/platform/pg-connection-scope" +import { cleanupTestDbs, createTestDb, executeSql, queryFirstRow, type TestDb } from "@/platform/test-pglite" +import { + loadOptionalFeatureState, + runWithDb, + type WorkflowStepLike, +} from "./ClickHouseSchemaApplyWorkflow.run" describe("loadOptionalFeatureState", () => { const successfulReads = { @@ -43,3 +51,59 @@ describe("loadOptionalFeatureState", () => { } }) }) + +describe("runWithDb failure bookkeeping", () => { + const trackedDbs: TestDb[] = [] + afterEach(() => cleanupTestDbs(trackedDbs)) + + /** Runs each durable step's callback directly — no retries, no persistence. */ + const inlineStep: WorkflowStepLike = { + do: ( + _name: string, + configOrCallback: { retries?: unknown } | (() => Promise), + callback?: () => Promise, + ): Promise => { + if (typeof configOrCallback === "function") return configOrCallback() + // The overload's trailing callback is absence, not a value to compare: + // a config with no callback is a malformed `step.do` call. + return Option.match(Option.fromUndefinedOr(callback), { + onNone: () => Promise.reject(new Error("missing step callback")), + onSome: (run) => run(), + }) + }, + } + + it("marks the run failed when config loading fails, instead of leaving it queued", async () => { + const testDb = createTestDb(trackedDbs) + const database = await Effect.runPromise(Effect.provide(Database, testDb.layer)) + const connection: PgConnectionScopeApi = { + run: (fn) => database.execute(fn), + close: () => Promise.resolve(), + } + // A queued claim exists, but the org's settings row does not (deleted + // after queueing) — loadConfig throws before any migration work starts. + await executeSql( + testDb, + `INSERT INTO org_clickhouse_schema_apply_runs (org_id, status, phase, created_at, updated_at) + VALUES ('org_wf_cfg', 'queued', 'queued', now(), now())`, + ) + + await expect( + runWithDb( + connection, + { MAPLE_DB: null, MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 5).toString("base64") }, + { payload: { orgId: "org_wf_cfg" } }, + inlineStep, + ), + ).rejects.toThrow("No ClickHouse settings configured") + + // Without the failed transition, OrgClickHouseSettingsService reads the + // leftover "queued" as already_running forever. + const row = await queryFirstRow<{ status: string; error_message: string | null }>( + testDb, + "SELECT status, error_message FROM org_clickhouse_schema_apply_runs WHERE org_id = 'org_wf_cfg'", + ) + expect(row?.status).toBe("failed") + expect(row?.error_message).toContain("No ClickHouse settings configured") + }) +}) diff --git a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts index 7b73b60be..97a3e61b4 100644 --- a/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts +++ b/apps/api/src/workflows/ClickHouseSchemaApplyWorkflow.run.ts @@ -390,7 +390,8 @@ export async function runClickHouseSchemaApply( } } -async function runWithDb( +/** Exported for tests: the orchestration body behind `runClickHouseSchemaApply`. */ +export async function runWithDb( connection: PgConnectionScopeApi, env: SchemaApplyWorkflowEnv, event: WorkflowEventLike, @@ -411,18 +412,27 @@ async function runWithDb( const appliedVersions: number[] = [] const skippedFeatures: Array<{ readonly id: string; readonly reason: string }> = [] - const cfg = await step.do("load-config", STEP, async () => { - const c = await loadConfig(dbStep, orgId, encryptionKey) - await updateRun( - dbStep, - orgId, - { status: "running", phase: "connecting", errorMessage: null, startedAt: new Date(startedAt) }, - Date.now(), - ) - return c - }) - + // Inside the protected region below: a config-load failure (settings row + // deleted, decrypt failure, invalid URL) must still transition the run row + // to failed, or the service reads the leftover "queued" as already_running + // and the org can never apply its schema again. try { + const cfg = await step.do("load-config", STEP, async () => { + const c = await loadConfig(dbStep, orgId, encryptionKey) + await updateRun( + dbStep, + orgId, + { + status: "running", + phase: "connecting", + errorMessage: null, + startedAt: new Date(startedAt), + }, + Date.now(), + ) + return c + }) + await step.do("ensure-bookkeeping", STEP, () => ensureMigrationsTable(cfg).then(() => undefined)) const applied = await step.do("read-applied", STEP, () => readAppliedVersions(cfg).then((s) => [...s]), diff --git a/apps/api/src/workflows/InvestigationFanoutWorkflow.run.test.ts b/apps/api/src/workflows/InvestigationFanoutWorkflow.run.test.ts index 1b8f59da2..3fdf4c1e4 100644 --- a/apps/api/src/workflows/InvestigationFanoutWorkflow.run.test.ts +++ b/apps/api/src/workflows/InvestigationFanoutWorkflow.run.test.ts @@ -630,4 +630,20 @@ describe("runInvestigationFanout", () => { expect((await run(baseDeps())).status).toBe("skipped") expect(await loadLanes()).toHaveLength(0) }) + + it("stands down a stale attempt instead of publishing over a restart", async () => { + // A restart bumped the fence and re-queued the row; the terminated-but-alive + // attempt-0 instance replays its claim. Termination is best-effort, so this + // check is the only thing keeping the old workflow from overwriting the new + // attempt's status, report, and lanes' parent state. + await harness.db + .update(investigations) + .set({ fanoutAttempt: 1, fanoutState: "queued" }) + .where(eq(investigations.id, harness.investigationId)) + expect((await run(baseDeps())).status).toBe("skipped") + const row = await loadInvestigation() + expect(row.status).toBe("investigating") + expect(row.reportJson).toBeNull() + expect(await loadLanes()).toHaveLength(0) + }) }) diff --git a/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts b/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts index 75af8748a..514fa38a3 100644 --- a/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts +++ b/apps/api/src/workflows/InvestigationFanoutWorkflow.run.ts @@ -664,6 +664,19 @@ async function runWithDb( ), ) + /** + * Every parent-row write carries the attempt fence: a straggler instance that + * outlived a best-effort termination — or replayed after a restart bumped + * `fanoutAttempt` — must find zero rows, not overwrite the live attempt's + * status, plan, or report. Lane rows are already attempt-scoped. + */ + const fencedParentRow = () => + and( + eq(investigations.orgId, orgIdTyped), + eq(investigations.id, idTyped), + eq(investigations.fanoutAttempt, attempt), + ) + const laneRows = () => dbStep((db) => db @@ -696,6 +709,11 @@ async function runWithDb( if (row.fanoutState !== "queued" && row.fanoutState !== "running") { return { proceed: false as const } } + // The attempt is a fencing token: a restart bumps `fanoutAttempt` and + // termination of the prior instance is best-effort, so an old instance + // replaying its claim against the restarted row must stand down rather + // than run to completion over the new attempt's state. + if (row.fanoutAttempt !== attempt) return { proceed: false as const } // Both deadlines fixed here, once, and returned on the cached result. Reading // a clock anywhere downstream of this would differ per replay. @@ -709,7 +727,7 @@ async function runWithDb( fanoutDeadlineAt: new Date(hypothesisDeadlineAtMs), updatedAt: new Date(now), }) - .where(and(eq(investigations.orgId, orgIdTyped), eq(investigations.id, idTyped))), + .where(fencedParentRow()), ) return { @@ -799,7 +817,7 @@ async function runWithDb( plannerElapsedMs: finishedAt - startedAt, updatedAt: new Date(finishedAt), }) - .where(and(eq(investigations.orgId, orgIdTyped), eq(investigations.id, idTyped))) + .where(fencedParentRow()) for (const [ordinal, hypothesis] of plan.hypotheses.entries()) { await db @@ -943,12 +961,7 @@ async function runWithDb( reportJson: output.report as never, updatedAt: new Date(finishedAt), }) - .where( - and( - eq(investigations.orgId, orgIdTyped), - eq(investigations.id, idTyped), - ), - ), + .where(fencedParentRow()), ) } return { @@ -1043,7 +1056,7 @@ async function runWithDb( outputTokens, updatedAt: new Date(now), }) - .where(and(eq(investigations.orgId, orgIdTyped), eq(investigations.id, idTyped))), + .where(fencedParentRow()), ) await meterTokens(env, orgId, investigationId, attempt, inputTokens, outputTokens) return { status: "inconclusive" as const } @@ -1120,7 +1133,7 @@ async function runWithDb( outputTokens, updatedAt: new Date(now), }) - .where(and(eq(investigations.orgId, orgIdTyped), eq(investigations.id, idTyped))), + .where(fencedParentRow()), ) await meterTokens(env, orgId, investigationId, attempt, inputTokens, outputTokens) return { metered: inputTokens + outputTokens } @@ -1135,7 +1148,7 @@ async function runWithDb( db .update(investigations) .set({ fanoutState: "validating", updatedAt: new Date(startedAt) }) - .where(and(eq(investigations.orgId, orgIdTyped), eq(investigations.id, idTyped))), + .where(fencedParentRow()), ) // Read the lanes back from Postgres rather than from the step results: a diff --git a/apps/cli/src/commands/server.ts b/apps/cli/src/commands/server.ts index 64d0db2ab..6f93367a2 100644 --- a/apps/cli/src/commands/server.ts +++ b/apps/cli/src/commands/server.ts @@ -3,7 +3,7 @@ import { FileSystem } from "effect/FileSystem" import * as Command from "effect/unstable/cli/Command" import * as Flag from "effect/unstable/cli/Flag" import { HttpClient } from "effect/unstable/http" -import { openSync } from "node:fs" +import { closeSync, openSync, writeSync } from "node:fs" import { homedir } from "node:os" import { dirname, join } from "node:path" import { startServer } from "../server/serve" @@ -154,6 +154,24 @@ const startBanner = ( // `maple stop` finds it without knowing the full data path. const pidFilePath = (dataDir: string): string => join(dirname(dataDir), "maple.pid") +/** Exclusively create the PID file (O_EXCL) so exactly one `maple start` can + * proceed past the liveness guard. Exported for the start-serialization test. */ +export const claimPidFileExclusive = (pidPath: string): Effect.Effect => + Effect.try({ + try: () => { + const fd = openSync(pidPath, "wx", 0o600) + writeSync(fd, String(process.pid)) + closeSync(fd) + }, + catch: (error) => + new ServerStateError({ + message: + (error as NodeJS.ErrnoException).code === "EEXIST" + ? "maple is already running or starting — stop it with `maple stop`" + : `could not claim the PID file at ${pidPath}: ${error instanceof Error ? error.message : String(error)}`, + }), + }) + /** Read the PID file, returning `none` when it is missing or unparseable. */ const readPid = (fs: FileSystem, pidPath: string): Effect.Effect> => fs.readFileString(pidPath).pipe( @@ -351,9 +369,7 @@ const BACKGROUND_READY_TIMEOUT_MS = BACKGROUND_READY_POLL_MS * BACKGROUND_READY_ * because a typo that quietly turned checkpointing off would reintroduce exactly * the data loss this exists to prevent. */ -export const parseCheckpointInterval = ( - value: string, -): Duration.Duration | undefined | "invalid" => { +export const parseCheckpointInterval = (value: string): Duration.Duration | undefined | "invalid" => { const raw = value.trim().toLowerCase() if (raw === "off" || raw === "0" || raw === "none") return undefined const match = raw.match(/^(\d+)\s*(s|m|h)$/) @@ -383,20 +399,15 @@ export const parseCheckpointInterval = ( * and has never been checkpointed, which is every existing install on the first * start after upgrading. */ -export const needsInitialCheckpoint = ( - availability: CheckpointAvailability, - hasLiveData: boolean, -): boolean => hasLiveData && !availability.available && availability.reason === "none" +export const needsInitialCheckpoint = (availability: CheckpointAvailability, hasLiveData: boolean): boolean => + hasLiveData && !availability.available && availability.reason === "none" /** * Does the store hold live data, as opposed to just the preserved checkpoint * registry? `backups` is skipped for the same reason it is skipped when the * live store is reset — it is not part of the data being protected. */ -const storeHasLiveData = ( - fs: FileSystem, - dataDir: string, -): Effect.Effect => +const storeHasLiveData = (fs: FileSystem, dataDir: string): Effect.Effect => fs.readDirectory(dataDir).pipe( Effect.map((entries) => entries.some((entry) => entry !== "backups")), Effect.orElseSucceed(() => false), @@ -500,7 +511,7 @@ const takeCheckpointQuietly = ( process.stderr.write( reason === "initial" ? `${green("✓")} checkpoint taken — recover an unclean shutdown with ` + - `${bold("maple restore --yes")}\n` + `${bold("maple restore --yes")}\n` : dim("◌ checkpoint refreshed\n"), ), ), @@ -509,7 +520,9 @@ const takeCheckpointQuietly = ( debugLog(`checkpoint (${reason}) failed`, error.message) process.stderr.write( reason === "initial" - ? dim(`◌ could not take an initial checkpoint — run ${bold("maple checkpoint")} to retry\n`) + ? dim( + `◌ could not take an initial checkpoint — run ${bold("maple checkpoint")} to retry\n`, + ) : dim("◌ could not refresh the checkpoint — the previous one still stands\n"), ) }), @@ -883,6 +896,16 @@ export const start = Command.make("start", { }), ) + // Claim the PID file EXCLUSIVELY before chDB or the listener opens. + // The liveness guard above is check-then-act: two concurrent starts + // could both pass it and open the same store natively, and the + // second would read the first's open sentinel as an unclean + // shutdown. O_EXCL makes exactly one start win; the loser exits + // with the ordinary already-running error before touching the store. + yield* Effect.acquireRelease(claimPidFileExclusive(pidPath), () => + fs.remove(pidPath, { force: true }).pipe(Effect.ignore), + ) + const { port: boundPort } = yield* startServer({ hostname: bindHost, browserHosts: Array.from( @@ -901,10 +924,6 @@ export const start = Command.make("start", { }) started = true - yield* Effect.acquireRelease(fs.writeFileString(pidPath, String(process.pid)), () => - fs.remove(pidPath, { force: true }).pipe(Effect.ignore), - ) - const bindAddr = serverUrl(bindHost, boundPort) const connectAddr = serverUrl(advertiseHost, boundPort) // Default: send users to the auto-updating UI on local.maple.dev (it diff --git a/apps/cli/src/core/update.ts b/apps/cli/src/core/update.ts index 935465032..67e852005 100644 --- a/apps/cli/src/core/update.ts +++ b/apps/cli/src/core/update.ts @@ -19,6 +19,7 @@ import { PlatformError } from "effect/PlatformError" import { HttpClient, HttpClientRequest } from "effect/unstable/http" import * as ChildProcess from "effect/unstable/process/ChildProcess" import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" +import { randomUUID } from "node:crypto" import { realpathSync } from "node:fs" import { dirname, join } from "node:path" import { amber, bold, dim, green } from "../lib/style" @@ -296,7 +297,43 @@ const clearQuarantine = (paths: ReadonlyArray): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem + const previousDir = join(tmpDir, "previous") + const mapleDst = join(installDir, "maple") + const libDst = join(installDir, "libchdb.so") + const restorePrevious = Effect.gen(function* () { + for (const [parked, dst] of [ + [join(previousDir, "maple"), mapleDst], + [join(previousDir, "libchdb.so"), libDst], + ] as const) { + if (yield* fs.exists(parked)) { + yield* fs.remove(dst, { force: true }).pipe(Effect.ignore) + yield* fs.rename(parked, dst) + } + } + }).pipe(Effect.ignore) + yield* Effect.gen(function* () { + yield* fs.makeDirectory(previousDir, { recursive: true }) + if (yield* fs.exists(mapleDst)) yield* fs.rename(mapleDst, join(previousDir, "maple")) + yield* fs.rename(join(srcDir, "maple"), mapleDst) + if (yield* fs.exists(libDst)) yield* fs.rename(libDst, join(previousDir, "libchdb.so")) + yield* fs.rename(join(srcDir, "libchdb.so"), libDst) + yield* fs.chmod(mapleDst, 0o755) + }).pipe(Effect.tapError(() => restorePrevious)) + }) export interface UpdateResult { readonly tag: string @@ -320,8 +357,26 @@ export const performUpdate = ( const name = `maple-${tag}-${target}` const url = `https://github.com/${REPO}/releases/download/${tag}/${name}.tar.gz` // Temp dir lives *inside* installDir so the final rename is same-filesystem - // (atomic; cross-device rename would EXDEV). - const tmpDir = join(installDir, ".maple-update-tmp") + // (atomic; cross-device rename would EXDEV). It is UNIQUE per invocation: + // a shared name let one updater delete another's extracted library after + // its executable had already been installed, pairing mismatched files. + const tmpDir = join(installDir, `.maple-update-tmp-${randomUUID()}`) + + // Best-effort sweep of temp dirs abandoned by crashed updates (including + // the legacy fixed ".maple-update-tmp" name). Age-gated so a concurrent + // in-flight update is never touched. + const sweepStaleTmpDirs = Effect.gen(function* () { + const entries = yield* fs.readDirectory(installDir) + const cutoff = Date.now() - 60 * 60 * 1000 + for (const entry of entries) { + if (!entry.startsWith(".maple-update-tmp")) continue + const path = join(installDir, entry) + const info = yield* fs.stat(path) + if (Option.exists(info.mtime, (mtime) => mtime.getTime() < cutoff)) { + yield* fs.remove(path, { recursive: true, force: true }) + } + } + }).pipe(Effect.ignore) yield* Effect.scoped( Effect.gen(function* () { @@ -329,11 +384,10 @@ export const performUpdate = ( fs.remove(tmpDir, { recursive: true, force: true }).pipe(Effect.ignore), ) - // Fresh temp dir. - yield* fs.remove(tmpDir, { recursive: true, force: true }).pipe( - Effect.andThen(fs.makeDirectory(tmpDir, { recursive: true })), - Effect.mapError((e) => mapFsError(e, installDir)), - ) + yield* sweepStaleTmpDirs + yield* fs + .makeDirectory(tmpDir, { recursive: true }) + .pipe(Effect.mapError((e) => mapFsError(e, installDir))) const tarball = join(tmpDir, "bundle.tar.gz") yield* downloadTo(url, tarball) @@ -351,10 +405,7 @@ export const performUpdate = ( yield* extractTar(tarball, tmpDir) const srcDir = join(tmpDir, name) - // Atomic in-place swap of both bundle files. - yield* fs.rename(join(srcDir, "maple"), join(installDir, "maple")).pipe( - Effect.andThen(fs.rename(join(srcDir, "libchdb.so"), join(installDir, "libchdb.so"))), - Effect.andThen(fs.chmod(join(installDir, "maple"), 0o755)), + yield* swapBundlePair(srcDir, installDir, tmpDir).pipe( Effect.mapError((e) => mapFsError(e, installDir)), ) @@ -422,3 +473,5 @@ export const maybeNotifyUpdate: Effect.Effect { }) continue } + // The pointer must select EXACTLY ONE verified generation. A stale but + // well-formed pointer (its target deleted or never published) would + // otherwise classify every real generation as superseded — and with + // keep=0 the whole range would enter the delete set while the pointer + // references nothing that survives. That is uncertain state: over-retain. + const pointerTargets = verified.filter((g) => g.generationId === activeGenerationId) + if (pointerTargets.length !== 1) { + excludedRanges.push({ + signal, + rangeStart, + reason: + `active pointer selects ${pointerTargets.length} verified generations ` + + `(target ${activeGenerationId}); range is uncertain (over-retained)`, + }) + continue + } // Partition: active (never deleted) vs superseded. const superseded = verified .filter((g) => g.generationId !== activeGenerationId) @@ -348,16 +365,14 @@ const readActiveGenerationIdStrict = ( if (!existsSync(pointerPath)) return null assertNoSymlinkSync(archiveDir, pointerPath, "archive active pointer") assertRealFileSync(pointerPath, "archive active pointer") - const raw = JSON.parse(readFileSync(pointerPath, "utf8")) as Record - if ( - raw.formatVersion !== 1 || - typeof raw.generationId !== "string" || - raw.signal !== signal || - raw.rangeStart !== rangeDate - ) { - throw new Error(`malformed active pointer at ${pointerPath}`) - } - return raw.generationId + // Full pointer validation (id shape, selectedAt, signal/range binding) — + // the same parser every other pointer read uses, not a looser private one. + const pointer = parseArchiveActivePointer( + JSON.parse(readFileSync(pointerPath, "utf8")) as unknown, + signal, + rangeDate, + ) + return pointer.generationId } /** Deterministic tombstone path for a GC target beneath the operation dir. */ diff --git a/apps/cli/src/server/archives/generation.ts b/apps/cli/src/server/archives/generation.ts index 58c679c3f..d4a8cdc8d 100644 --- a/apps/cli/src/server/archives/generation.ts +++ b/apps/cli/src/server/archives/generation.ts @@ -1062,6 +1062,13 @@ const reconcilePrePublication = async ( building: string, endPhase: ArchiveOperationPhase, ): Promise => { + // Durably record the terminal phase BEFORE any cleanup. The pin release + // below is phase-gated on recovery: releasing it first and crashing before + // this write would leave a mid-flight phase whose required pin is absent, + // which validateOwnedPinState rightly fails closed on — stranding every + // later archive operation. From "aborted", each cleanup step is idempotent + // and decideCreate resumes this same path. + await advancePhase(archiveDir, intent.operationId, endPhase) // Quarantine incomplete building output if present (retain, don't delete). if (existsSync(building)) { // Move the building debris into a quarantine subdir named for the @@ -1096,7 +1103,6 @@ const reconcilePrePublication = async ( // "pin-released" would be an error, but a pre-publication abort releasing its // own pin is the intended recovery — so tolerate already-absent here. await releaseOwnedPin(dataDir, intent) - await advancePhase(archiveDir, intent.operationId, endPhase) // Archive the aborted operation journal to completed/ (retained for audit). await archiveCompletedOperation(archiveDir, intent.operationId) } diff --git a/apps/cli/src/server/archives/reconcile.ts b/apps/cli/src/server/archives/reconcile.ts index 1757e639e..d4846fd3f 100644 --- a/apps/cli/src/server/archives/reconcile.ts +++ b/apps/cli/src/server/archives/reconcile.ts @@ -193,9 +193,28 @@ const decideCreate = ( if (!snapshot.promoted && phaseAtLeast(phase, "promoted") && phase !== "aborted") { return { kind: "FailClosed", reason: `phase ${phase} requires its final generation`, operationId } } - // 5. aborted operation still in the active directory. + // 5. A durably aborted operation still in the active directory. This is a + // LEGITIMATE crash state: the abort path writes "aborted" durably and then + // finishes cleanup + archival, so a crash anywhere in that tail leaves an + // aborted journal here. Resume the idempotent abort finish — but only when + // nothing was published; an aborted phase alongside a final generation is a + // contradiction and fails closed. if (phase === "aborted") { - return { kind: "FailClosed", reason: "aborted operation still in active dir", operationId } + if (snapshot.promoted) { + return { + kind: "FailClosed", + reason: "aborted operation has a published final generation", + operationId, + } + } + return { + kind: "CreateAbortPrepublication", + operationId, + journalDigest, + migrationRequired, + intent, + buildingPresent: snapshot.buildingPresent, + } } // Terminal verify-only: phase >= complete. diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index a9ce98594..c2e34b719 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -463,7 +463,8 @@ const localQueryError = (status: number, detail: string, cause = detail): LocalQ export const checkpointQueryUrl = (host: string, port: number): string => `${serverUrl(host, port)}/local/query` -const postCheckpointBackup = ( +/** Exported for the timeout regression test only. */ +export const postCheckpointBackup = ( host: string, port: number, dataDir: string, @@ -494,12 +495,13 @@ const postCheckpointBackup = ( try: () => JSON.parse(responseText) as unknown, catch: (error) => localQueryError(response.status, errorMessage(error), errorCause(error)), }) - }).pipe( - Effect.timeout("30 seconds"), - Effect.catchTag("TimeoutError", () => - Effect.fail(localQueryError(0, "local checkpoint backup timed out after 30 seconds")), - ), - ) + }) + // Deliberately NO client-side timeout. The server runs BACKUP through a + // synchronous db.exec that cannot observe cancellation, so a timeout here + // unwound checkpoint creation and released the maintenance lock while chDB + // was still writing the snapshot — a retry would then quarantine/rename the + // directory a live BACKUP was writing into. A large store legitimately backs + // up for minutes; a dead server surfaces as a connection error instead. } export const isMissingBackupConfigurationError = (error: unknown): boolean => { diff --git a/apps/cli/src/server/durable-files.ts b/apps/cli/src/server/durable-files.ts index 685fd67de..227a79e79 100644 --- a/apps/cli/src/server/durable-files.ts +++ b/apps/cli/src/server/durable-files.ts @@ -29,6 +29,22 @@ export const isUnsupportedDirectorySyncError = (error: unknown): boolean => unsupportedDirectorySyncCodes.has(String((error as NodeJS.ErrnoException).code)) export const ensurePrivateDirectory = async (path: string): Promise => { + await ensureRealDirectory(path) + await chmod(path, 0o700) +} + +/** Like {@link ensurePrivateDirectory}, but an already-existing directory keeps + * its mode. `durableWrite` targets sit beside caller-chosen paths (markers and + * journals next to an arbitrary `--data-dir`), so hardening the parent would + * chmod e.g. `/var/lib` to 0700 and break every other service under it. Only + * a directory this call creates gets 0700. */ +export const ensureParentDirectory = async (path: string): Promise => { + const existed = await ensureRealDirectory(path) + if (!existed) await chmod(path, 0o700) +} + +/** Returns whether the directory already existed. */ +const ensureRealDirectory = async (path: string): Promise => { const before = await lstat(path).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null throw error @@ -40,7 +56,7 @@ export const ensurePrivateDirectory = async (path: string): Promise => { if (after.isSymbolicLink() || !after.isDirectory()) { throw new Error(`private directory is not a real directory: ${path}`) } - await chmod(path, 0o700) + return before !== null } export const syncDirectory = async (path: string, faults: DurabilityFaults = {}): Promise => { @@ -62,7 +78,7 @@ export const durableWrite = async ( faults: DurabilityFaults = {}, ): Promise => { const parent = dirname(path) - await ensurePrivateDirectory(parent) + await ensureParentDirectory(parent) const temporary = join(parent, `.${randomUUID()}.tmp`) const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) try { diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 12526a1d3..9abcb6312 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -33,7 +33,7 @@ import { type StoreMarker, type StoreMarkerV2, } from "./store-version" -import { durableJson, durableRename, ensurePrivateDirectory } from "./durable-files" +import { durableJson, durableRename, ensurePrivateDirectory, syncTree } from "./durable-files" import { MAPLE_VERSION } from "../version" import { decodeMigrationJournal, type MigrationJournalSchema } from "./local-store-migrations/journal-schema" import { legacyToCurrentModule } from "./local-store-migrations/legacy-to-current" @@ -893,6 +893,13 @@ export const promoteLocalStoreMigration = async ( await assertRealDirectory(targetData, "migration target") const marker = validateTargetMarker(readMarker(journal.targetDataDir), "staging") await assertRealFile(stagedMarker, "staged target marker") + // Cloned target contents were written with plain `fs.cp` and never fsynced; + // durableRename below only makes the DIRECTORY ENTRIES durable. Without this, + // a power loss after cutover can leave a durable active pointer naming a + // store whose file contents never reached stable storage. Runs before the + // renames so a crash-and-resume repeats it. Store trees carry + // engine-managed symlinks, hence allowSymlinks. + await syncTree(targetData, { allowSymlinks: true }) if (!sourceExists) { await assertRealDirectory(activeData, "source data") @@ -1409,23 +1416,6 @@ export const runLocalStoreMigration = async ( if (isStoreDirty(dataDir)) throw new Error("source store was not cleanly closed; preserve it and retry after inspection") - if (journal === null) { - if (!plan || plan.chain.length === 0) throw new Error("store already has the current schema") - journal = createJournal(dataDir, plan, markerState!) - await ensurePrivateDirectory(migrationRootPath(dataDir, journal.migrationId)) - await ensurePrivateDirectory(join(migrationRootPath(dataDir, journal.migrationId), "target")) - if (existsSync(journal.targetDataDir)) - throw new Error( - "migration target directory already exists without a journal; refusing to reuse it", - ) - await writeMigrationJournal(dataDir, journal) - } else { - assertJournalPaths(dataDir, journal) - journalModules(journal, localStoreMigrations) - if (markerState?.formatVersion === 2 && markerState.storeId !== journal.sourceStoreId) - throw new Error("unfinished local migration source store id does not match the active marker") - } - const operationId = randomUUID() return withMigrationMaintenanceLock(dataDir, operationId, async () => { assertNoLiveServer(dataDir) @@ -1433,7 +1423,33 @@ export const runLocalStoreMigration = async ( throw new Error( "source store became dirty before the migration lock was acquired; refusing to continue", ) - let current = (await readMigrationJournal(dataDir)) ?? journal! + // The canonical journal is created, validated, and written only UNDER the + // maintenance lock. The unlocked read above informed planning only: two + // concurrent `schema migrate --yes` runs would otherwise both observe "no + // journal", write competing journals with different migration ids, and + // leave one executing a transaction the other's journal describes. + let current = await readMigrationJournal(dataDir) + if (current === null) { + if (!plan || plan.chain.length === 0) throw new Error("store already has the current schema") + if (markerState === null) + throw new Error( + "the source store has no readable marker; unknown stores fail closed and cannot be migrated", + ) + const created = createJournal(dataDir, plan, markerState) + await ensurePrivateDirectory(migrationRootPath(dataDir, created.migrationId)) + await ensurePrivateDirectory(join(migrationRootPath(dataDir, created.migrationId), "target")) + if (existsSync(created.targetDataDir)) + throw new Error( + "migration target directory already exists without a journal; refusing to reuse it", + ) + await writeMigrationJournal(dataDir, created) + current = created + } else { + assertJournalPaths(dataDir, current) + journalModules(current, localStoreMigrations) + if (markerState?.formatVersion === 2 && markerState.storeId !== current.sourceStoreId) + throw new Error("unfinished local migration source store id does not match the active marker") + } try { const modules = journalModules(current, localStoreMigrations) current = await executeMigrationChain(dataDir, current, modules, options.onProgress) diff --git a/apps/cli/src/server/local-store-migrations/journal-codecs.ts b/apps/cli/src/server/local-store-migrations/journal-codecs.ts index b2615af33..a2cd7ab75 100644 --- a/apps/cli/src/server/local-store-migrations/journal-codecs.ts +++ b/apps/cli/src/server/local-store-migrations/journal-codecs.ts @@ -12,6 +12,8 @@ // `Schema.decodeUnknownEffect` is a one-line swap if that runner ever moves // into Effect. import { Schema } from "effect" +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, join, sep } from "node:path" import { RAW_TELEMETRY_TTL_COLUMNS, type Chdb } from "../chdb" import { decodeTableRowCounts } from "../chdb-rows" import { withRawTelemetryRetentionFloor, type LocalSchemaManifest } from "../schema-manifest" @@ -120,3 +122,23 @@ export const expectedManifest = ( retentionDays === undefined ? manifest : withRawTelemetryRetentionFloor(manifest, RAW_TABLES_INTERNAL, retentionDays) + +/** + * Clone a clean, stopped store into a staged migration target — WITHOUT its + * checkpoint registry. `/backups` belongs to the retained source: its + * manifests pin the source's schema fingerprint, so a copied registry fails + * every post-promotion resolution against the new fingerprint, classifying the + * registry "unusable" and blocking the fresh checkpoint the migration tells + * the user to create. Checkpoints stay with the rollback source, as the stated + * preservation envelope already promises. + */ +export const cloneStoreForStaging = async (source: string, target: string): Promise => { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + const checkpointRoot = join(source, "backups") + await cp(source, target, { + recursive: true, + preserveTimestamps: true, + filter: (src) => src !== checkpointRoot && !src.startsWith(`${checkpointRoot}${sep}`), + }) +} diff --git a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts index fddec8fd4..665508a65 100644 --- a/apps/cli/src/server/local-store-migrations/legacy-to-current.ts +++ b/apps/cli/src/server/local-store-migrations/legacy-to-current.ts @@ -528,6 +528,34 @@ const duplicateGroupCount = async ( return rows[0] === undefined ? 0 : Number(rows[0].rowCount) } +/** + * Replay SELECTs re-enable quoted 64-bit JSON output for THIS query only. The + * shared connection pins `output_format_json_quote_64bit_integers=0`, under + * which raw UInt64 data columns (trace Duration, histogram counts and + * Array(UInt64) buckets) decode as lossy JS numbers above 2^53 — the copy + * would then reinsert a silently different integer. Quoted output round-trips + * exactly: JSONEachRow input accepts quoted 64-bit integers, arrays included. + */ +const EXACT_INT64_SETTINGS = "SETTINGS output_format_json_quote_64bit_integers = 1" + +/** First fetch of a table is capped small: the byte budget is applied only + * AFTER a batch is materialized and decoded, so `batchRows` full rows land in + * memory first. With multi-megabyte log bodies that is gigabytes; seed + * conservatively and let the observed average row size grow the limit. */ +const INITIAL_FETCH_ROWS = 128 + +export const nextFetchRowLimit = ( + table: Pick, + fetchedRows: number, + fetchedBytes: number, +): number => { + if (fetchedRows === 0 || fetchedBytes <= 0) return INITIAL_FETCH_ROWS + const averageRowBytes = fetchedBytes / fetchedRows + // Aim slightly past the byte budget so a typical batch still fills it. + const target = Math.ceil((table.batchBytes * 1.25) / averageRowBytes) + return Math.max(1, Math.min(table.batchRows, target)) +} + const copyTable = async ( context: MigrationModuleContext, table: LegacyRawTable, @@ -536,6 +564,7 @@ const copyTable = async ( ): Promise => { let current = initial let progress = copyProgressFor(current, table.name) + let fetchRows = Math.min(table.batchRows, INITIAL_FETCH_ROWS) const columnList = columns.map((column) => identifier(column.name)).join(", ") const hashExpression = `cityHash64(toString(tuple(${columnList})))` const tieBreakExpression = `sipHash64(toString(tuple(${columnList})))` @@ -563,9 +592,10 @@ const copyTable = async ( const offset = continuation.offset === 0 ? "" : ` OFFSET ${continuation.offset}` const output = await querySource( context, - `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${table.batchRows}${offset}`, + `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${fetchRows}${offset} ${EXACT_INT64_SETTINGS}`, ) const rawRows = decodeJsonObjectRows(output) + fetchRows = nextFetchRowLimit(table, rawRows.length, Buffer.byteLength(output)) let rows = rawRows if (rows.length === 0) break const candidates = rows.map((row) => { @@ -662,7 +692,10 @@ const recoverPendingBatch = async ( const inserted = decodeJsonObjectRows( await queryTarget( context, - `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${pending.rowCount}${offset}`, + // Same quoted-64-bit output as the source SELECT: the recovered batch + // signature is computed over the re-encoded rows and must match byte + // for byte. + `SELECT ${columnList}, toString(${timeNs}) AS __maple_timestamp, toString(${hashExpression}) AS __maple_hash, toString(${tieBreakExpression}) AS __maple_tie_break FROM ${identifier(table.name)} WHERE ${identifier(table.timeColumn)} >= ${timestampLiteral(retentionStartAt(context.cutoffAt, table.retentionDays))} AND ${identifier(table.timeColumn)} <= ${timestampLiteral(context.cutoffAt)} ${cursor} ORDER BY ${timeNs}, ${hashExpression}, ${tieBreakExpression} LIMIT ${pending.rowCount}${offset} ${EXACT_INT64_SETTINGS}`, ), ) if (inserted.length !== pending.rowCount) @@ -928,3 +961,6 @@ export const legacyToCurrentModule: LocalStoreMigrationModule => { +const prepareTarget = async ( + context: MigrationModuleContext, + state: V10ToV11State, +): Promise => { await context.closeStores() const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v11-to-v12-service-map-edge-quantiles.ts b/apps/cli/src/server/local-store-migrations/v11-to-v12-service-map-edge-quantiles.ts index 2e8d6f4e2..96432229c 100644 --- a/apps/cli/src/server/local-store-migrations/v11-to-v12-service-map-edge-quantiles.ts +++ b/apps/cli/src/server/local-store-migrations/v11-to-v12-service-map-edge-quantiles.ts @@ -1,6 +1,6 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { cloneStoreForStaging } from "./journal-codecs" +import { resolve } from "node:path" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -185,9 +185,7 @@ const prepareTarget = async ( const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v12-to-v13-service-operations-discriminators.ts b/apps/cli/src/server/local-store-migrations/v12-to-v13-service-operations-discriminators.ts index 25d8a0ccb..9e411c5c2 100644 --- a/apps/cli/src/server/local-store-migrations/v12-to-v13-service-operations-discriminators.ts +++ b/apps/cli/src/server/local-store-migrations/v12-to-v13-service-operations-discriminators.ts @@ -1,6 +1,6 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { cloneStoreForStaging } from "./journal-codecs" +import { resolve } from "node:path" import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" import type { LocalStoreMigrationModule, @@ -186,9 +186,7 @@ const prepareTarget = async ( const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts index 10e4e60ac..476d21b34 100644 --- a/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts +++ b/apps/cli/src/server/local-store-migrations/v2-to-v3-service-map-ingest-bridge.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -60,9 +60,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V2ToV3State const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts index 3a9af3138..e1a823fa7 100644 --- a/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts +++ b/apps/cli/src/server/local-store-migrations/v3-to-v4-web-events.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -60,9 +60,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V3ToV4State const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts index e0429be07..40f97aff2 100644 --- a/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts +++ b/apps/cli/src/server/local-store-migrations/v4-to-v5-service-overview-minutely.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -60,9 +60,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V4ToV5State const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts index 4818e3f45..6d9c2148e 100644 --- a/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts +++ b/apps/cli/src/server/local-store-migrations/v5-to-v6-error-events-fingerprint-hygiene.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -60,9 +60,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V5ToV6State const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts index 4045f5d6a..5f2819f0a 100644 --- a/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts +++ b/apps/cli/src/server/local-store-migrations/v6-to-v7-error-service-version.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -79,9 +79,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V6ToV7State const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts index 197dffb88..5184731c1 100644 --- a/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts +++ b/apps/cli/src/server/local-store-migrations/v7-to-v8-apple-crash-frames.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -60,9 +60,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V7ToV8State const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v8-to-v9-mv-sweep.ts b/apps/cli/src/server/local-store-migrations/v8-to-v9-mv-sweep.ts index 84aa1a1a5..ffcb1cee8 100644 --- a/apps/cli/src/server/local-store-migrations/v8-to-v9-mv-sweep.ts +++ b/apps/cli/src/server/local-store-migrations/v8-to-v9-mv-sweep.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -100,9 +100,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V8ToV9State const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/src/server/local-store-migrations/v9-to-v10-semconv-key-renames.ts b/apps/cli/src/server/local-store-migrations/v9-to-v10-semconv-key-renames.ts index 793f46574..f7f7b02ec 100644 --- a/apps/cli/src/server/local-store-migrations/v9-to-v10-semconv-key-renames.ts +++ b/apps/cli/src/server/local-store-migrations/v9-to-v10-semconv-key-renames.ts @@ -1,7 +1,7 @@ // SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. -import { cp, mkdir, rm } from "node:fs/promises" -import { dirname, resolve } from "node:path" +import { resolve } from "node:path" import { + cloneStoreForStaging, decodeInstalledProgress, makeRawRowsState, type InstalledProgress, @@ -88,9 +88,7 @@ const prepareTarget = async (context: MigrationModuleContext, state: V9ToV10Stat const source = resolve(context.sourceDataDir) const target = resolve(context.targetDataDir) if (source !== target) { - await rm(target, { recursive: true, force: true }) - await mkdir(dirname(target), { recursive: true, mode: 0o700 }) - await cp(source, target, { recursive: true, preserveTimestamps: true }) + await cloneStoreForStaging(source, target) } return state } diff --git a/apps/cli/test/archive-gc.test.ts b/apps/cli/test/archive-gc.test.ts index 2767ac53a..0f46f3399 100644 --- a/apps/cli/test/archive-gc.test.ts +++ b/apps/cli/test/archive-gc.test.ts @@ -228,6 +228,41 @@ describe("archive gc planning", () => { }) }) + it("over-retains a range whose well-formed pointer selects no verified generation", async () => { + await withArchive(async (archiveDir) => { + // Both generations are real and verifiable, but the pointer — valid in + // shape, binding, and selectedAt — targets a generation that does not + // exist. Without the exactly-one-verified-target invariant, both + // generations would be classified superseded and keep=0 would delete + // the ENTIRE range while the pointer references nothing that survives. + await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) + await seedPublishedGeneration(archiveDir, { + createdAt: "2026-06-03T00:00:00.000Z", + selectActive: true, + }) + await rebuildSignalCatalog(archiveDir, "traces") + const pointerPath = activePointerPath(archiveDir, "traces", "2026-06-01") + writeFileSync( + pointerPath, + JSON.stringify({ + formatVersion: 1, + generationId: randomUUID(), + signal: "traces", + rangeStart: "2026-06-01", + selectedAt: "2026-06-04T00:00:00.000Z", + }), + ) + const plan = planArchiveGc(archiveDir, 0) + strictEqual(plan.deleteSet.length, 0, "nothing targeted under a stale pointer") + ok( + plan.excludedRanges.some( + (r) => r.rangeStart === "2026-06-01" && r.reason.includes("selects 0 verified"), + ), + "range excluded as uncertain", + ) + }) + }) + it("excludes an entire signal when a generation manifest is malformed", async () => { await withArchive(async (archiveDir) => { await seedPublishedGeneration(archiveDir, { createdAt: "2026-06-02T00:00:00.000Z" }) diff --git a/apps/cli/test/archive-reconcile-decision.test.ts b/apps/cli/test/archive-reconcile-decision.test.ts index 5b791551a..3f3a33b64 100644 --- a/apps/cli/test/archive-reconcile-decision.test.ts +++ b/apps/cli/test/archive-reconcile-decision.test.ts @@ -225,9 +225,20 @@ describe("decideReconciliation — CREATE impossible-topology FailClosed gates", strictEqual(d.kind, "FailClosed") }) - it("aborted in active/ → FailClosed", () => { + it("aborted in active/ (not promoted) → resume the idempotent abort finish", () => { + // A durably aborted journal in active/ is the LEGITIMATE crash state + // between the abort path's phase write and its archival — it must resume, + // not strand every later archive operation behind FailClosed. const intent = createIntent("aborted") const d = decideReconciliation(valid(createSnapshot(intent, { promoted: false }))) + strictEqual(d.kind, "CreateAbortPrepublication") + }) + + it("aborted in active/ with a published generation → FailClosed", () => { + const intent = createIntent("aborted") + const d = decideReconciliation( + valid(createSnapshot(intent, { promoted: true, manifestAtFinal: true })), + ) strictEqual(d.kind, "FailClosed") }) }) diff --git a/apps/cli/test/archive-reconcile.test.ts b/apps/cli/test/archive-reconcile.test.ts index d328d8d93..f208426dc 100644 --- a/apps/cli/test/archive-reconcile.test.ts +++ b/apps/cli/test/archive-reconcile.test.ts @@ -1169,3 +1169,107 @@ function seedGcOpWithEvidence( ) return { opId, opDir } } + +describe("durably aborted operation recovery", () => { + const seedAbortedOperation = ( + archiveDir: string, + dataDir: string, + scratchRoot: string, + opts: { pinPresent: boolean; scratchPresent: boolean; phase?: string }, + ) => { + const opId = randomUUID() + const gid = randomUUID() + const pinId = randomUUID() + const checkpointId = randomUUID() + const opDir = join(archiveDir, "operations", "active", `archive-${opId}`) + mkdirSync(opDir, { recursive: true }) + if (opts.pinPresent) { + const pinDir = join(dataDir, "backups", "pins", checkpointId) + mkdirSync(pinDir, { recursive: true }) + writeFileSync( + join(pinDir, `${pinId}.json`), + JSON.stringify({ + formatVersion: 1, + pinId, + checkpointId, + purpose: `archive:${gid}`, + createdAt: "2026-06-01T00:00:00.000Z", + }), + ) + } + const scratchSubdir = `archive-${opId}` + if (opts.scratchPresent) mkdirSync(join(scratchRoot, scratchSubdir), { recursive: true }) + writeFileSync( + join(opDir, "intent.json"), + JSON.stringify({ + formatVersion: 3, + kind: "create", + operationId: opId, + generationId: gid, + signal: "traces", + rangeStart: "2026-06-01", + checkpointId, + archiveDir, + dataDir, + scratchRoot, + pinId, + pinPurpose: `archive:${gid}`, + scratchSubdir, + manifestSha256: null, + baseActiveGenerationId: null, + phase: opts.phase ?? "aborted", + createdAt: "2026-06-01T00:00:00.000Z", + updatedAt: "2026-06-01T00:00:00.000Z", + }), + ) + return { opId, opDir, pinPath: join(dataDir, "backups", "pins", checkpointId, `${pinId}.json`) } + } + + it("finishes an aborted journal left in active/ (crash between abort write and cleanup)", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const seeded = seedAbortedOperation(archiveDir, dataDir, scratchRoot, { + pinPresent: true, + scratchPresent: true, + }) + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }) + strictEqual(existsSync(seeded.opDir), false, "aborted journal archived out of active/") + ok( + existsSync(join(archiveDir, "operations", "completed", `archive-${seeded.opId}`)), + "aborted journal retained under completed/", + ) + strictEqual(existsSync(seeded.pinPath), false, "owned pin released") + strictEqual( + existsSync(join(scratchRoot, `archive-${seeded.opId}`)), + false, + "owned scratch removed", + ) + }) + }) + + it("finishes an aborted journal whose pin and scratch were already cleaned", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + const seeded = seedAbortedOperation(archiveDir, dataDir, scratchRoot, { + pinPresent: false, + scratchPresent: false, + }) + await runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }) + strictEqual(existsSync(seeded.opDir), false, "aborted journal archived out of active/") + }) + }) + + it("still fails closed when a mid-flight phase has lost its pin", async () => { + await withRoots(async (archiveDir, dataDir, scratchRoot) => { + // Pin absent at a phase where release was never authorized: this is NOT + // the abort crash state and must stay FailClosed. + seedAbortedOperation(archiveDir, dataDir, scratchRoot, { + pinPresent: false, + scratchPresent: false, + phase: "shards-written", + }) + await rejects( + runArchiveReconciliation(dataDir, archiveDir, scratchRoot, { dryRun: false }), + /pin is missing before release was authorized/, + ) + }) + }) +}) diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 42d74dff1..a0fcaa0a0 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,6 +1,7 @@ // BOUNDARY: Test doubles preserve opaque values so the consuming boundary can be exercised. import { describe, it } from "@effect/vitest" -import { Effect, Exit, Option } from "effect" +import { Clock, Duration, Effect, Exit, Option } from "effect" +import { HttpClient, HttpClientResponse } from "effect/unstable/http" import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" import { existsSync, @@ -31,6 +32,7 @@ import { newCheckpointOperationId, newCheckpointQuarantineId, parseCheckpointManifest, + postCheckpointBackup, parseCheckpointState, readCheckpointState, reconcileCheckpointRecovery, @@ -1073,3 +1075,48 @@ const throwsMessage = (run: () => unknown, expected: RegExp): void => { match(error instanceof Error ? error.message : String(error), expected) } } + +describe("checkpoint backup request has no client-side timeout", () => { + // The server executes BACKUP through a synchronous db.exec that cannot + // observe client cancellation. A client timeout unwound checkpoint creation + // and released the maintenance lock while the snapshot was still being + // written — so a legitimately slow BACKUP must be waited out, not raced. + // + // Simulated time: every Clock.sleep runs 1000x faster, so the removed 30s + // ceiling would fire at ~30ms real time while the stub server answers at + // ~600ms — the old code deterministically failed here with its TimeoutError. + const scaledClock: Clock.Clock = { + currentTimeMillisUnsafe: () => Date.now(), + currentTimeMillis: Effect.sync(() => Date.now()), + currentTimeNanosUnsafe: () => BigInt(Date.now()) * 1_000_000n, + currentTimeNanos: Effect.sync(() => BigInt(Date.now()) * 1_000_000n), + monotonicTimeNanosUnsafe: () => process.hrtime.bigint(), + monotonicTimeNanos: Effect.sync(() => process.hrtime.bigint()), + sleep: (duration) => + Effect.promise(() => new Promise((r) => setTimeout(r, Duration.toMillis(duration) / 1000))), + } + + it("waits out a backup slower than the old 30s ceiling", async () => { + const root = mkdtempSync(join(tmpdir(), "maple-backup-timeout-")) + const dataDir = join(root, "data") + mkdirSync(dataDir, { recursive: true }) + writeFileSync(`${dataDir}.maintenance-token`, "token\n") + const slowClient = HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb(request, new Response('{"ok":true}', { status: 200 })), + ).pipe(Effect.delay("10 minutes")), + ) + try { + const exit = await Effect.runPromise( + postCheckpointBackup("127.0.0.1", 4318, dataDir, newCheckpointId()).pipe( + Effect.exit, + Effect.provideService(HttpClient.HttpClient, slowClient), + Effect.provideService(Clock.Clock, scaledClock), + ), + ) + ok(Exit.isSuccess(exit), `slow backup must succeed, got ${JSON.stringify(exit)}`) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/cli/test/durable-files.test.ts b/apps/cli/test/durable-files.test.ts new file mode 100644 index 000000000..c812bf554 --- /dev/null +++ b/apps/cli/test/durable-files.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest" +import { mkdirSync, mkdtempSync, rmSync, statSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { durableWrite, ensurePrivateDirectory } from "../src/server/durable-files" + +const withRoot = async (run: (root: string) => Promise): Promise => { + const root = mkdtempSync(join(tmpdir(), "maple-durable-files-test-")) + try { + await run(root) + } finally { + rmSync(root, { recursive: true, force: true }) + } +} + +const modeOf = (path: string): number => statSync(path).mode & 0o777 + +describe("durableWrite parent handling", () => { + it("preserves the mode of an existing parent directory", async () => { + await withRoot(async (root) => { + // A caller-owned parent (e.g. /var/lib for --data-dir /var/lib/maple): + // sibling markers/journals land here and must never re-permission it. + const parent = join(root, "shared") + mkdirSync(parent, { mode: 0o755 }) + await durableWrite(join(parent, "maple-store-version.json"), "{}\n") + expect(modeOf(parent)).toBe(0o755) + }) + }) + + it("still creates a missing parent as private 0700", async () => { + await withRoot(async (root) => { + const parent = join(root, "created", "nested") + await durableWrite(join(parent, "state.json"), "{}\n") + expect(modeOf(parent)).toBe(0o700) + }) + }) + + it("ensurePrivateDirectory keeps hardening explicit Maple-owned roots", async () => { + await withRoot(async (root) => { + const owned = join(root, "backups") + mkdirSync(owned, { mode: 0o755 }) + await ensurePrivateDirectory(owned) + expect(modeOf(owned)).toBe(0o700) + }) + }) +}) diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 1697abd0b..717b09a09 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -57,9 +57,13 @@ import { import { ensureStoreMarkerDurable, readMarker, storeMarkerPath } from "../src/server/store-version" import { durableJson } from "../src/server/durable-files" import { + __testables as legacyTestables, advanceDuplicateKeyProgress, duplicateCursorContinuation, + LEGACY_RAW_TABLES, + nextFetchRowLimit, type CopyProgress, + type RawReplayProgress, } from "../src/server/local-store-migrations/legacy-to-current" import { v10ToV11ProductEventsModule } from "../src/server/local-store-migrations/v10-to-v11-product-events" import { v11ToV12ServiceMapEdgeQuantilesModule } from "../src/server/local-store-migrations/v11-to-v12-service-map-edge-quantiles" @@ -1397,3 +1401,158 @@ describe("v10 -> v11 product events module", () => { ).resolves.toEqual({ state, progress }) }) }) + +describe("legacy raw replay fetch bounds", () => { + const tracesTable = LEGACY_RAW_TABLES[1] + + it("seeds the first fetch small instead of materializing batchRows full rows", () => { + expect(nextFetchRowLimit(tracesTable, 0, 0)).toBe(128) + }) + + it("grows toward batchRows for small rows and shrinks for huge rows", () => { + // 128 rows of ~200 bytes: the budget allows far more — clamp to batchRows. + expect(nextFetchRowLimit(tracesTable, 128, 128 * 200)).toBe(tracesTable.batchRows) + // 4 rows of ~8 MiB: even one row overshoots the budget — floor at 1. + expect(nextFetchRowLimit(tracesTable, 4, 4 * 8 * 1024 * 1024)).toBe(1) + // ~64 KiB rows: the limit lands near budget/rowBytes, never above batchRows. + const limit = nextFetchRowLimit(tracesTable, 100, 100 * 64 * 1024) + expect(limit).toBeGreaterThanOrEqual(64) + expect(limit).toBeLessThanOrEqual(128) + }) +}) + +describe("legacy raw replay 64-bit exactness", () => { + it("requests quoted 64-bit output and reinserts a >2^53 UInt64 verbatim", async () => { + const bigDuration = "9007199254740993" // 2^53 + 1: rounds to ...992 as a JS number + const sourceQueries: string[] = [] + const targetStatements: string[] = [] + let call = 0 + const fakeSourceDb = { + query: (sql: string): string => { + sourceQueries.push(sql) + call += 1 + if (call > 1) return "" + return `${JSON.stringify({ + Timestamp: "2026-08-30 12:00:00.000000000", + Duration: bigDuration, + __maple_timestamp: "1756555200000000000", + __maple_hash: "18446744073709551615", + __maple_tie_break: "3", + })}\n` + }, + } + const fakeTargetDb = { + query: (sql: string): string => { + targetStatements.push(sql) + return "" + }, + exec: (sql: string): void => { + targetStatements.push(sql) + }, + } + const context = { + dataDir: "/tmp/fake", + sourceDataDir: "/tmp/fake-source", + targetDataDir: "/tmp/fake-target", + source: LEGACY_LOCAL_SCHEMA, + target: LOCAL_SCHEMA_V1, + cutoffAt: "2026-08-31T00:00:00.000Z", + step: { + id: "local-0000-to-0001-raw-replay", + moduleVersion: 1, + from: LEGACY_LOCAL_SCHEMA, + to: LOCAL_SCHEMA_V1, + status: "running" as const, + }, + openSource: async (fn: (db: typeof fakeSourceDb) => string | Promise) => fn(fakeSourceDb), + openTarget: async (fn: (db: typeof fakeTargetDb) => string | void | Promise) => + fn(fakeTargetDb), + closeStores: async () => undefined, + ensureCapacity: async () => undefined, + saveStep: async () => undefined, + } as MigrationModuleContext + const columns = [ + { name: "Timestamp", type: "DateTime64(9)" }, + { name: "Duration", type: "UInt64" }, + ] + const initial: RawReplayProgress = { sourceInventory: {}, copied: {} } + await legacyTestables.copyTable(context, LEGACY_RAW_TABLES[1], columns, initial) + + // The source SELECT must override the connection-wide unquoted 64-bit + // output; without it chDB emits Duration as a JSON number and the decode + // below would round it before reinsertion. + expect(sourceQueries[0]).toContain("SETTINGS output_format_json_quote_64bit_integers = 1") + const insert = targetStatements.find((sql) => sql.startsWith("INSERT INTO")) + expect(insert).toBeDefined() + expect(insert).toContain(`"Duration":"${bigDuration}"`) + }) +}) + +describe("clone-based staging excludes the checkpoint registry", () => { + it("clones store contents but never /backups", async () => { + const root = await mkdtemp(join(tmpdir(), "maple-clone-staging-")) + try { + const source = join(root, "source") + const target = join(root, "target", "data") + await mkdir(join(source, "store", "parts"), { recursive: true }) + await mkdir(join(source, "backups", "snapshots", "cp-1"), { recursive: true }) + const { writeFile } = await import("node:fs/promises") + await writeFile(join(source, "store", "parts", "part.bin"), "data") + await writeFile(join(source, "backups", "state.json"), "{}") + await writeFile( + join(source, "backups", "snapshots", "cp-1", "manifest.json"), + // A copied manifest pins the OLD schema fingerprint: post-promotion it + // fails resolution and marks the registry "unusable", blocking the new + // checkpoint the migration instructs the user to create. + JSON.stringify({ schemaFingerprint: "stale" }), + ) + const { cloneStoreForStaging } = + await import("../src/server/local-store-migrations/journal-codecs") + await cloneStoreForStaging(source, target) + const { existsSync } = await import("node:fs") + expect(existsSync(join(target, "store", "parts", "part.bin"))).toBe(true) + expect(existsSync(join(target, "backups"))).toBe(false) + // The registry stays with the retained rollback source. + expect(existsSync(join(source, "backups", "state.json"))).toBe(true) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) +}) + +describe("migration journal creation is lock-serialized", () => { + it("does not create a journal while another maintenance operation holds the lock", async () => { + const root = await mkdtemp(join(tmpdir(), "maple-migration-lock-order-")) + try { + const dataDir = join(root, "data") + await mkdir(join(dataDir, "store"), { recursive: true }) + const { writeFile } = await import("node:fs/promises") + await writeFile( + storeMarkerPath(dataDir), + `${JSON.stringify({ + chdb: (await import("../src/version")).CHDB_VERSION, + maple: "test", + createdAt: "2026-08-30T00:00:00.000Z", + schema: LEGACY_LOCAL_SCHEMA.fingerprint, + })}\n`, + ) + const { withMaintenanceLock } = await import("../src/server/checkpoints") + const { randomUUID } = await import("node:crypto") + await withMaintenanceLock(dataDir, randomUUID(), async () => { + // A concurrent migrate must fail at the lock WITHOUT having written + // the canonical journal first — journal creation used to happen + // before lock acquisition and could clobber a running migration's + // journal with a fresh one under a different migration id. + await expect(runLocalStoreMigration({ dataDir })).rejects.toThrow( + /another Maple maintenance operation is active/, + ) + }) + const { existsSync } = await import("node:fs") + expect(existsSync(migrationJournalPath(dataDir))).toBe(false) + // And no orphaned migration root either. + expect(existsSync(join(root, ".maple-migrations"))).toBe(false) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/cli/test/native-archive-calibrate-probe.sh b/apps/cli/test/native-archive-calibrate-probe.sh index 57a9773d6..53a2763f5 100755 --- a/apps/cli/test/native-archive-calibrate-probe.sh +++ b/apps/cli/test/native-archive-calibrate-probe.sh @@ -109,10 +109,14 @@ stop_server() { } insert_markers() { - # Insert enough rows per signal (>= 2*SAMPLE_ROWS=10, so 30 each) so - # calibration has a disjoint held-out window AND a representative set. + # Calibration splits rows [0, SAMPLE_ROWS) for training and + # [SAMPLE_ROWS, SAMPLE_ROWS * (1 + HELD_OUT_SAMPLE_MULTIPLIER)) for held-out + # validation, so with --sample-rows 10 and a multiplier of 2 it needs 30 rows + # per signal, not the 20 an earlier comment here claimed. Seeding exactly 30 + # left no margin: any row the split could not place failed every candidate at + # once, reported as "insufficient for a complete six-signal held-out split". local i - for i in $(seq 0 29); do + for i in $(seq 0 44); do local sec min t sec=$(printf '%02d' $((i % 60))) min=$(printf '%02d' $((i / 60))) diff --git a/apps/cli/test/server-pid.test.ts b/apps/cli/test/server-pid.test.ts new file mode 100644 index 000000000..4e90f1ad1 --- /dev/null +++ b/apps/cli/test/server-pid.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest" +import { Effect, Exit } from "effect" +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { claimPidFileExclusive } from "../src/commands/server" + +describe("start PID claim is exclusive", () => { + it("exactly one claimant wins; the loser fails without touching the file", async () => { + const root = mkdtempSync(join(tmpdir(), "maple-pid-claim-")) + try { + const pidPath = join(root, "maple.pid") + await Effect.runPromise(claimPidFileExclusive(pidPath)) + expect(readFileSync(pidPath, "utf8")).toBe(String(process.pid)) + // A racing second start must fail with the already-running refusal — + // before this claim existed, both starts passed the read-then-check + // guard and raced to open the same chDB store. + const second = await Effect.runPromiseExit(claimPidFileExclusive(pidPath)) + expect(Exit.isFailure(second)).toBe(true) + expect(JSON.stringify(second)).toContain("already running or starting") + expect(readFileSync(pidPath, "utf8")).toBe(String(process.pid)) + expect(existsSync(pidPath)).toBe(true) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/cli/test/update.test.ts b/apps/cli/test/update.test.ts index 5d3a8a3ea..9b6d8d4d0 100644 --- a/apps/cli/test/update.test.ts +++ b/apps/cli/test/update.test.ts @@ -6,7 +6,7 @@ import { mkdtemp, readFile, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" import * as BunServices from "@effect/platform-bun/BunServices" -import { Duration, Effect } from "effect" +import { Duration, Effect, Exit } from "effect" import { FileSystem } from "effect/FileSystem" import { FetchHttpClient } from "effect/unstable/http" import { @@ -252,3 +252,53 @@ describe("mapFsError", () => { strictEqual(__testables.mapFsError(new Error("disk on fire"), "/tmp/x").message, "disk on fire") }) }) + +describe("swapBundlePair", () => { + const layout = async () => { + const { mkdtemp, mkdir, writeFile } = await import("node:fs/promises") + const { tmpdir } = await import("node:os") + const root = await mkdtemp(join(tmpdir(), "maple-update-swap-")) + const installDir = join(root, "install") + const srcDir = join(root, "src") + const tmpDir = join(root, "tmp") + await mkdir(installDir, { recursive: true }) + await mkdir(srcDir, { recursive: true }) + await mkdir(tmpDir, { recursive: true }) + await writeFile(join(installDir, "maple"), "old-maple") + await writeFile(join(installDir, "libchdb.so"), "old-lib") + return { root, installDir, srcDir, tmpDir } + } + + // Plain `it` + `Effect.runPromise` (see archive-candidate-child.test.ts): + // `it.effect` hangs on real fs work under bun test. + it("installs both files together", () => + Effect.runPromise( + Effect.gen(function* () { + const { root, installDir, srcDir, tmpDir } = yield* Effect.promise(layout) + const fs = yield* FileSystem + yield* fs.writeFileString(join(srcDir, "maple"), "new-maple") + yield* fs.writeFileString(join(srcDir, "libchdb.so"), "new-lib") + yield* __testables.swapBundlePair(srcDir, installDir, tmpDir) + strictEqual(yield* fs.readFileString(join(installDir, "maple")), "new-maple") + strictEqual(yield* fs.readFileString(join(installDir, "libchdb.so")), "new-lib") + yield* fs.remove(root, { recursive: true, force: true }) + }).pipe(Effect.provide(BunServices.layer)), + )) + + it("restores the matched old pair when the second rename fails", () => + Effect.runPromise( + Effect.gen(function* () { + const { root, installDir, srcDir, tmpDir } = yield* Effect.promise(layout) + const fs = yield* FileSystem + // Only the executable extracted — the library rename will fail after + // the maple swap already happened. The old code left new-maple beside + // old-lib; the swap must put the matched old pair back instead. + yield* fs.writeFileString(join(srcDir, "maple"), "new-maple") + const exit = yield* __testables.swapBundlePair(srcDir, installDir, tmpDir).pipe(Effect.exit) + ok(Exit.isFailure(exit), "swap must report the failure") + strictEqual(yield* fs.readFileString(join(installDir, "maple")), "old-maple") + strictEqual(yield* fs.readFileString(join(installDir, "libchdb.so")), "old-lib") + yield* fs.remove(root, { recursive: true, force: true }) + }).pipe(Effect.provide(BunServices.layer)), + )) +}) diff --git a/apps/local-ui/src/hooks/use-local-metric-detail.test.ts b/apps/local-ui/src/hooks/use-local-metric-detail.test.ts new file mode 100644 index 000000000..6db31f714 --- /dev/null +++ b/apps/local-ui/src/hooks/use-local-metric-detail.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest" +import { Effect } from "effect" +import { + compileMetricRateTimeseriesQuery, + compileMetricValueTimeseriesQuery, +} from "./use-local-metric-detail" + +const params = { + orgId: "org_local", + startTime: "2026-07-30 13:05:00", + endTime: "2026-07-30 14:05:00", + bucketSeconds: 60, + metricName: "http.server.duration", +} + +describe("metric detail timeseries queries", () => { + // The chart draws at most 60 series; without the top-N cap a + // high-cardinality install fetches and pivots every service's series before + // the render limit ever applies. + it("caps the value timeseries to the chart's series budget in the query", () => { + const { sql } = Effect.runSync(compileMetricValueTimeseriesQuery({ metricType: "gauge" }, params)) + expect(sql).toContain("WITH __series_base AS") + expect(sql).toContain("LIMIT 60") + }) + + it("caps the rate timeseries the same way", () => { + const { sql } = Effect.runSync( + compileMetricRateTimeseriesQuery( + { metricName: "http.server.duration", bucketSeconds: 60 }, + params, + ), + ) + expect(sql).toContain("WITH __series_base AS") + expect(sql).toContain("LIMIT 60") + }) +}) diff --git a/apps/local-ui/src/hooks/use-local-metric-detail.ts b/apps/local-ui/src/hooks/use-local-metric-detail.ts index eb80f8cde..30c209f5b 100644 --- a/apps/local-ui/src/hooks/use-local-metric-detail.ts +++ b/apps/local-ui/src/hooks/use-local-metric-detail.ts @@ -1,4 +1,6 @@ import { keepPreviousData, useQuery } from "@tanstack/react-query" +import { Option } from "effect" +import { HARD_SERIES_LIMIT } from "@maple/ui/components/plot" import { CH } from "@maple/query-engine" import { executeLocalCompiledQuery } from "@/lib/query" import { LOCAL_ORG_ID } from "../lib/constants" @@ -42,6 +44,24 @@ export interface MetricSeriesPoint { value: number } +/** + * The detail chart draws at most `HARD_SERIES_LIMIT` lines, so cap the series + * in the query — a high-cardinality install would otherwise fetch and pivot + * every service's series only for the chart to drop all but 60 of them. + * Exported through the compile helpers below so the cap is testable. + */ +const SERIES_CAP = { groupBy: ["service"], seriesLimit: HARD_SERIES_LIMIT } + +export const compileMetricRateTimeseriesQuery = ( + opts: { metricName: string; bucketSeconds: number }, + params: Parameters[1], +) => CH.compile(CH.metricsTimeseriesRateQuery({ ...opts, ...SERIES_CAP }), params) + +export const compileMetricValueTimeseriesQuery = ( + opts: { metricType: CH.MetricsTimeseriesOpts["metricType"] }, + params: Parameters[1], +) => CH.compile(CH.metricsTimeseriesQuery({ ...opts, ...SERIES_CAP }), params) + /** * Detail timeseries, one series per service. Monotonic counters plot the true * per-second rate (window-CTE query); gauges/histograms plot the average value. @@ -53,37 +73,46 @@ export function useLocalMetricTimeseries(entry: MetricEntry | null | undefined, queryKey: ["local", "metrics", "timeseries", metricName, entry?.metricType, isRate, range], enabled: entry != null, placeholderData: keepPreviousData, - queryFn: async (): Promise> => { - const { startTime, endTime } = boundsForRange(range) - const bucketSeconds = bucketSecondsForRange(range) - const params = { orgId: LOCAL_ORG_ID, startTime, endTime, bucketSeconds, metricName: metricName! } - if (isRate) { - const rows = await executeLocalCompiledQuery( - CH.compile( - CH.metricsTimeseriesRateQuery({ metricName: metricName!, bucketSeconds }), - params, - ), - ) - return rows.map((r) => ({ - bucket: r.bucket, - groupName: r.groupName, - value: Number(r.rateValue), - })) - } - const rows = await executeLocalCompiledQuery( - CH.compile( - CH.metricsTimeseriesQuery({ - metricType: entry!.metricType as CH.MetricsTimeseriesOpts["metricType"], - }), - params, - ), - ) - return rows.map((r) => ({ - bucket: r.bucket, - groupName: r.groupName, - value: Number(r.avgValue), - })) - }, + queryFn: (): Promise> => + Option.match(Option.fromNullishOr(entry), { + // Unreachable: `enabled` gates the query on the entry existing. + onNone: () => Promise.resolve([]), + onSome: async (metric) => { + const { startTime, endTime } = boundsForRange(range) + const bucketSeconds = bucketSecondsForRange(range) + const params = { + orgId: LOCAL_ORG_ID, + startTime, + endTime, + bucketSeconds, + metricName: metric.metricName, + } + if (isRate) { + const rows = await executeLocalCompiledQuery( + compileMetricRateTimeseriesQuery( + { metricName: metric.metricName, bucketSeconds }, + params, + ), + ) + return rows.map((r) => ({ + bucket: r.bucket, + groupName: r.groupName, + value: Number(r.rateValue), + })) + } + const rows = await executeLocalCompiledQuery( + compileMetricValueTimeseriesQuery( + { metricType: metric.metricType as CH.MetricsTimeseriesOpts["metricType"] }, + params, + ), + ) + return rows.map((r) => ({ + bucket: r.bucket, + groupName: r.groupName, + value: Number(r.avgValue), + })) + }, + }), }) } diff --git a/apps/scraper/src/Env.test.ts b/apps/scraper/src/Env.test.ts new file mode 100644 index 000000000..42d2eed45 --- /dev/null +++ b/apps/scraper/src/Env.test.ts @@ -0,0 +1,67 @@ +import { assert, describe, it } from "@effect/vitest" +import { ConfigProvider, Effect } from "effect" +import { ScraperEnv } from "./Env" + +const loadEnv = (env: Record) => + Effect.service(ScraperEnv).pipe( + Effect.provide(ScraperEnv.layer), + Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromUnknown(env)), + ) + +describe("ScraperEnv", () => { + it.effect("applies defaults when nothing is set", () => + Effect.gen(function* () { + const env = yield* loadEnv({}) + assert.strictEqual(env.SCRAPER_CONCURRENCY, 10) + assert.strictEqual(env.SCRAPER_RECONCILE_INTERVAL_SECONDS, 60) + assert.strictEqual(env.SCRAPER_OTLP_MAX_DATA_POINTS, 10_000) + assert.strictEqual(env.PORT, 3475) + }), + ) + + it.effect("accepts explicit valid numeric settings", () => + Effect.gen(function* () { + const env = yield* loadEnv({ SCRAPER_CONCURRENCY: "25", PORT: "8080" }) + assert.strictEqual(env.SCRAPER_CONCURRENCY, 25) + assert.strictEqual(env.PORT, 8080) + }), + ) + + // SCRAPER_CONCURRENCY=0 built a semaphore that could never grant the one + // permit each scrape requests — every target silently suspended while + // /health kept returning 200. Startup is the only visible place to fail. + it.effect("rejects a zero concurrency at layer build", () => + Effect.gen(function* () { + const error = yield* loadEnv({ SCRAPER_CONCURRENCY: "0" }).pipe(Effect.flip) + assert.strictEqual(error._tag, "ConfigError") + }), + ) + + it.effect("rejects a fractional concurrency at layer build", () => + Effect.gen(function* () { + const error = yield* loadEnv({ SCRAPER_CONCURRENCY: "0.5" }).pipe(Effect.flip) + assert.strictEqual(error._tag, "ConfigError") + }), + ) + + it.effect("rejects a non-positive reconcile interval", () => + Effect.gen(function* () { + const error = yield* loadEnv({ SCRAPER_RECONCILE_INTERVAL_SECONDS: "-1" }).pipe(Effect.flip) + assert.strictEqual(error._tag, "ConfigError") + }), + ) + + it.effect("rejects a non-positive OTLP chunk size", () => + Effect.gen(function* () { + const error = yield* loadEnv({ SCRAPER_OTLP_MAX_DATA_POINTS: "0" }).pipe(Effect.flip) + assert.strictEqual(error._tag, "ConfigError") + }), + ) + + it.effect("rejects a port outside the TCP range", () => + Effect.gen(function* () { + const error = yield* loadEnv({ PORT: "70000" }).pipe(Effect.flip) + assert.strictEqual(error._tag, "ConfigError") + }), + ) +}) diff --git a/apps/scraper/src/Env.ts b/apps/scraper/src/Env.ts index 22d0bc75f..7249baaed 100644 --- a/apps/scraper/src/Env.ts +++ b/apps/scraper/src/Env.ts @@ -1,4 +1,4 @@ -import { Config, Context, Effect, Layer, Redacted } from "effect" +import { Config, Context, Effect, Layer, Redacted, Schema } from "effect" export interface ScraperEnvConfig { /** Base URL of the Maple API, e.g. `https://api.maple.dev`. */ @@ -31,23 +31,33 @@ export interface ScraperEnvConfig { // configuration instead of crashing the turbo dev TUI. Production overrides // all three (see apps/scraper/railway.json deploy notes); a missing override // degrades to visible per-reconcile warnings, never a crash loop. +// +// Numeric settings ARE validated at layer build, unlike the URLs: a missing +// URL fails loudly every reconcile, but SCRAPER_CONCURRENCY=0 builds a +// semaphore that never grants a permit — every target silently suspended +// while /health keeps answering 200 — and a fractional or non-positive +// interval/chunk size misbehaves just as quietly. Failing startup is the +// only visible place for those. +const positiveInt = (name: string, maximum: number) => + Config.schema(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum })), name) + const envConfig = Config.all({ MAPLE_API_URL: Config.string("MAPLE_API_URL").pipe(Config.withDefault("http://127.0.0.1:3472")), SD_INTERNAL_TOKEN: Config.redacted("SD_INTERNAL_TOKEN").pipe( Config.withDefault(Redacted.make("maple-sd-dev-token")), ), MAPLE_INGEST_URL: Config.string("MAPLE_INGEST_URL").pipe(Config.withDefault("http://127.0.0.1:3474")), - SCRAPER_CONCURRENCY: Config.number("SCRAPER_CONCURRENCY").pipe(Config.withDefault(10)), - SCRAPER_RECONCILE_INTERVAL_SECONDS: Config.number("SCRAPER_RECONCILE_INTERVAL_SECONDS").pipe( + SCRAPER_CONCURRENCY: positiveInt("SCRAPER_CONCURRENCY", 10_000).pipe(Config.withDefault(10)), + SCRAPER_RECONCILE_INTERVAL_SECONDS: positiveInt("SCRAPER_RECONCILE_INTERVAL_SECONDS", 24 * 60 * 60).pipe( Config.withDefault(60), ), // 10k points is ~2 MB of OTLP/JSON at the attribute density Prometheus // exporters produce — a 10x margin under the gateway's 20 MB limit, so // even an unusually attribute-heavy exporter stays inside it. - SCRAPER_OTLP_MAX_DATA_POINTS: Config.number("SCRAPER_OTLP_MAX_DATA_POINTS").pipe( + SCRAPER_OTLP_MAX_DATA_POINTS: positiveInt("SCRAPER_OTLP_MAX_DATA_POINTS", 10_000_000).pipe( Config.withDefault(10_000), ), - PORT: Config.number("PORT").pipe(Config.withDefault(3475)), + PORT: positiveInt("PORT", 65_535).pipe(Config.withDefault(3475)), }) export class ScraperEnv extends Context.Service()("@maple/scraper/Env", { diff --git a/apps/scraper/src/OtlpIngest.test.ts b/apps/scraper/src/OtlpIngest.test.ts index f95a93dcf..71e5917e0 100644 --- a/apps/scraper/src/OtlpIngest.test.ts +++ b/apps/scraper/src/OtlpIngest.test.ts @@ -1,5 +1,6 @@ import { assert, describe, it } from "@effect/vitest" -import { Effect, Exit, Layer, Redacted } from "effect" +import { Effect, Exit, Fiber, Layer, Redacted } from "effect" +import { TestClock } from "effect/testing" import { FetchHttpClient } from "effect/unstable/http" import { OtlpIngest } from "./OtlpIngest" import { ScraperEnv, type ScraperEnvConfig } from "./Env" @@ -169,6 +170,29 @@ describe("OtlpIngest", () => { assert.isTrue(Exit.isFailure(spans[0]!.exit)) }).pipe(Effect.provide(TestLayer)), ) + // A gateway that accepts the connection but never answers must not pin the + // scrape (and its global concurrency permit) forever — the send times out + // as a retryable typed error. + it.effect("times out a stalled gateway request as a typed error", () => + Effect.gen(function* () { + const otlp = yield* OtlpIngest + // Bun's `fetch` type carries `preconnect`; nothing under test calls it. + const stalled: typeof globalThis.fetch = Object.assign(() => new Promise(() => {}), { + preconnect: () => Promise.resolve(), + }) + const fiber = yield* Effect.forkChild( + otlp + .send("maple_pk_test_key", SAMPLE_REQUEST) + .pipe(Effect.provideService(FetchHttpClient.Fetch, stalled), Effect.flip), + { startImmediately: true }, + ) + yield* TestClock.adjust("31 seconds") + const error = yield* Fiber.join(fiber) + assert.strictEqual(error._tag, "@maple/scraper/OtlpIngestError") + assert.strictEqual(error.status, null) + }).pipe(Effect.provide(TestLayer)), + ) + describe("chunked delivery", () => { /** Budget of 2 data points per POST, so a 5-point export needs 3 requests. */ const ChunkedLayer = OtlpIngest.layer.pipe( diff --git a/apps/scraper/src/OtlpIngest.ts b/apps/scraper/src/OtlpIngest.ts index 5ccb7412c..1a420340a 100644 --- a/apps/scraper/src/OtlpIngest.ts +++ b/apps/scraper/src/OtlpIngest.ts @@ -1,4 +1,4 @@ -import { Context, Effect, Layer, Schema } from "effect" +import { Context, Duration, Effect, Layer, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" import { countDataPoints, splitExportRequest, type OtlpExportRequest } from "./prometheus/otlp" import { ScraperEnv } from "./Env" @@ -28,6 +28,14 @@ export class OtlpIngest extends Context.Service()("@m const env = yield* ScraperEnv const client = yield* HttpClient.HttpClient + // Bound every chunk round-trip, like ApiClient does: `FetchHttpClient` + // sets no timeout, and each scrape holds a global concurrency permit + // through delivery — a gateway that accepts connections but never + // answers would otherwise pin permits until SCRAPER_CONCURRENCY stalled + // sends halt every target, with /health still reporting ok. A timeout + // fails the chunk as a retryable OtlpIngestError instead. + const REQUEST_TIMEOUT = Duration.seconds(30) + /** * Resolves to `null` on success, or to the error for a rejection the span * must NOT be blamed for. Carrying a 4xx out of the span as a *value* is @@ -45,6 +53,7 @@ export class OtlpIngest extends Context.Service()("@m const response = yield* client.execute(httpRequest).pipe( Effect.annotateSpans("peer.service", "ingest"), + Effect.timeout(REQUEST_TIMEOUT), Effect.mapError( (error) => new OtlpIngestError({ @@ -54,7 +63,12 @@ export class OtlpIngest extends Context.Service()("@m ), ) if (response.status < 200 || response.status >= 300) { - const text = yield* response.text.pipe(Effect.orElseSucceed(() => "")) + // The body read is bounded too — a stalled response stream after + // the status line would otherwise hang past the request timeout. + const text = yield* response.text.pipe( + Effect.timeout(REQUEST_TIMEOUT), + Effect.orElseSucceed(() => ""), + ) const error = new OtlpIngestError({ message: response.status === 402 diff --git a/apps/slack-agent/agent/lib/thread-follow-up.test.ts b/apps/slack-agent/agent/lib/thread-follow-up.test.ts index cc82d7467..6117af06f 100644 --- a/apps/slack-agent/agent/lib/thread-follow-up.test.ts +++ b/apps/slack-agent/agent/lib/thread-follow-up.test.ts @@ -398,6 +398,38 @@ describe("failing open", () => { }) }) + test("a truncated page suspends both bounds even when engagement IS visible", () => { + // The page is the oldest-first START of a longer thread, so the bot's + // post at its head says nothing about how buried it is now, and the + // page's last message is not the reply's predecessor. Computing the + // trailing-window bound from this prefix used to drop the follow-up as + // "engagement-buried" in a thread whose tail we cannot see. + const chatter = Array.from({ length: 48 }, (_, i) => + humanMessage(`chatter ${i}`, `1700000${String(100 + i).padStart(3, "0")}.000100`), + ) + const pending = promote(envelope({ event: { ts: "1700000200.000200" } }))! + expect(confirmThreadFollowUp(pending, [botMessage("1700000001.000100"), ...chatter])).toEqual({ + engaged: true, + reason: "page-truncated", + }) + }) + + test("a truncated page cannot declare the thread dormant off its stale tail", () => { + // The visible page ends days before the reply because the page is full, + // not because the thread went quiet — the actual predecessor is past the + // page. Measuring dormancy against the prefix's last message dropped + // live long threads. + const chatter = Array.from({ length: 48 }, (_, i) => + humanMessage(`chatter ${i}`, `1700000${String(100 + i).padStart(3, "0")}.000100`), + ) + // 25h after the last VISIBLE message. + const pending = promote(envelope({ event: { ts: "1700090148.000200" } }))! + expect(confirmThreadFollowUp(pending, [botMessage("1700000001.000100"), ...chatter])).toEqual({ + engaged: true, + reason: "page-truncated", + }) + }) + test("a short page with no engagement is a real answer, not a missing one", () => { const shortThread = Array.from({ length: 48 }, (_, i) => humanMessage(`chatter ${i}`, `1700000${String(100 + i).padStart(3, "0")}.000100`), diff --git a/apps/slack-agent/agent/lib/thread-follow-up.ts b/apps/slack-agent/agent/lib/thread-follow-up.ts index 892289af7..43fc1186c 100644 --- a/apps/slack-agent/agent/lib/thread-follow-up.ts +++ b/apps/slack-agent/agent/lib/thread-follow-up.ts @@ -453,11 +453,20 @@ export function confirmThreadFollowUp( return { engaged: true, reason: "thread-unreadable" } } + // Truncation is checked BEFORE either bound, not only when no engagement is + // visible: a full page is the oldest-first *start* of a longer thread, so + // its last element is not the reply's predecessor and its length is not the + // distance to the engagement. Computing the bounds anyway once declared an + // active incident thread dormant off the stale 50th message (a wrong drop — + // the expensive direction), and would keep an engagement parked at the end + // of the visible prefix "recent" forever (a wrong promote). Neither bound is + // answerable from this page; fail open, per the docblock above. + if (messages.length >= EVE_THREAD_PAGE_SIZE - 1) { + return { engaged: true, reason: "page-truncated" } + } + const engagementIndex = lastEngagementIndex(messages, pending.botUserId) if (engagementIndex === -1) { - if (messages.length >= EVE_THREAD_PAGE_SIZE - 1) { - return { engaged: true, reason: "page-truncated" } - } return { engaged: false, workedInThread: false, reason: "never-engaged" } } diff --git a/apps/web/src/components/dashboard-builder/sections/section-layout.test.ts b/apps/web/src/components/dashboard-builder/sections/section-layout.test.ts index 50082b354..5c6fd7a77 100644 --- a/apps/web/src/components/dashboard-builder/sections/section-layout.test.ts +++ b/apps/web/src/components/dashboard-builder/sections/section-layout.test.ts @@ -192,3 +192,20 @@ describe("autoLayoutPerContainer", () => { expect(autoLayoutPerContainer(widgets, sections).map((w) => w.id)).toEqual(["root", "grouped"]) }) }) + +describe("autoLayoutPerContainer with duplicate widget ids", () => { + it("keeps both widgets distinct instead of collapsing onto the last one", () => { + // Stored widget ids are not validated unique — an imported or API-written + // board can carry duplicates. An id-keyed map used to persist one widget's + // config twice and silently lose the other's. + const sections = [section("s1", ["t1"])] + const root = widget("dup", { x: 3, y: 3, w: 6, h: 4 }) + const grouped = widget("dup", { x: 5, y: 5, w: 4, h: 2 }, { sectionId: "s1", tabId: "t1" }) + const relaid = autoLayoutPerContainer([root, grouped], sections) + + expect(relaid).toHaveLength(2) + const layouts = relaid.map((w) => ({ ...w.layout, sectionId: w.sectionId })) + expect(layouts).toContainEqual({ x: 0, y: 0, w: 6, h: 4, sectionId: undefined }) + expect(layouts).toContainEqual({ x: 0, y: 0, w: 4, h: 2, sectionId: "s1" }) + }) +}) diff --git a/apps/web/src/components/dashboard-builder/sections/section-layout.ts b/apps/web/src/components/dashboard-builder/sections/section-layout.ts index 55a90d0b6..1594ce7e9 100644 --- a/apps/web/src/components/dashboard-builder/sections/section-layout.ts +++ b/apps/web/src/components/dashboard-builder/sections/section-layout.ts @@ -115,7 +115,10 @@ export function autoLayoutPerContainer( else byContainer.set(key, [widget]) } - const relaid = new Map() + // Keyed by object identity, not `widget.id`: stored widget IDs are not + // validated unique, and an id-keyed map made every duplicate collapse onto + // the last widget — persisting one config twice and losing the other. + const relaid = new Map() for (const bucket of byContainer.values()) { let currentX = 0 let currentY = 0 @@ -128,7 +131,7 @@ export function autoLayoutPerContainer( currentY += rowHeight rowHeight = 0 } - relaid.set(widget.id, { ...widget, layout: { ...widget.layout, x: currentX, y: currentY } }) + relaid.set(widget, { ...widget, layout: { ...widget.layout, x: currentX, y: currentY } }) currentX += w rowHeight = Math.max(rowHeight, h) } @@ -136,5 +139,5 @@ export function autoLayoutPerContainer( // Rebuild in the board's sorted order so array order still matches visual // order for the compactor. - return sortWidgetsForLayout(widgets, sections).map((widget) => relaid.get(widget.id) ?? widget) + return sortWidgetsForLayout(widgets, sections).map((widget) => relaid.get(widget) ?? widget) } diff --git a/apps/web/src/components/settings/roll-api-key-dialog.test.tsx b/apps/web/src/components/settings/roll-api-key-dialog.test.tsx new file mode 100644 index 000000000..499b7c291 --- /dev/null +++ b/apps/web/src/components/settings/roll-api-key-dialog.test.tsx @@ -0,0 +1,86 @@ +// @vitest-environment jsdom +// TEST-SEAM: the atom client and mutation-sync hooks are process-global wiring +// with no instance-level injection seam, so they are replaced at the module +// boundary and the dialog is exercised through its rendered controls. +import { cleanup, fireEvent, render, screen } from "@testing-library/react" +import { Exit, Schema } from "effect" +import { afterEach, describe, expect, it, vi } from "vitest" +import { V2ApiKey } from "@maple/domain/http/v2" + +/** What the roll mutation resolves to — only the fields this dialog reads. */ +interface RolledKey { + readonly secret: string + readonly txid: string +} +type RollResult = Exit.Exit + +const rollMutation = vi.fn<() => Promise>() +vi.mock("@/lib/effect-atom", () => ({ useAtomSet: () => rollMutation })) +vi.mock("@/lib/services/common/v2-atom-client", () => ({ + MapleApiV2AtomClient: { mutation: () => ({}) }, +})) +vi.mock("@/hooks/use-api-keys", () => ({ + useApiKeyMutationSync: () => ({ + prepareForMutation: () => {}, + reconcileTxid: async () => {}, + }), +})) +vi.mock("./api-key-secret-reveal", () => ({ + ApiKeySecretReveal: ({ secret }: { secret: string }) =>
{secret}
, +})) + +import { RollApiKeyDialog } from "./roll-api-key-dialog" + +const apiKey = Schema.decodeUnknownSync(V2ApiKey)({ + id: "key_aXwpxqBkqtYwtBtmsGbR41", + object: "api_key", + name: "ci-pipeline", + description: null, + key_prefix: "maple_ak_9f2c", + kind: "standard", + scopes: null, + revoked: false, + revoked_at: null, + last_used_at: null, + expires_at: null, + created_at: "2026-07-01T12:00:00.000Z", + created_by: "user_2Nk8mXqPfR3yZ1aB4cD5eF6g", + created_by_email: null, +}) + +afterEach(() => { + cleanup() + vi.clearAllMocks() +}) + +describe("RollApiKeyDialog", () => { + it("refuses every dismissal path while the roll is in flight, then reveals the secret", async () => { + let resolveRoll: ((value: RollResult) => void) | undefined + rollMutation.mockReturnValue( + new Promise((resolve) => { + resolveRoll = resolve + }), + ) + const onOpenChange = vi.fn() + render() + + fireEvent.click(screen.getByRole("button", { name: "Roll key" })) + + // The server may already have revoked the old key: closing here (the + // built-in close button, Escape) would drop the one-time replacement + // secret, or hand it to the next key's dialog. + fireEvent.click(screen.getByRole("button", { name: "Close" })) + fireEvent.keyDown(document.body, { key: "Escape" }) + expect(onOpenChange).not.toHaveBeenCalledWith(false) + + resolveRoll?.(Exit.succeed({ secret: "maple_ak_new_secret", txid: "42" })) + expect((await screen.findByTestId("secret")).textContent).toBe("maple_ak_new_secret") + + // Once the secret is on screen, closing works again (footer + built-in + // close button both render now — either dismisses). + const closeButton = screen.getAllByRole("button", { name: "Close" })[0] + expect(closeButton).toBeDefined() + if (closeButton) fireEvent.click(closeButton) + expect(onOpenChange).toHaveBeenCalledWith(false) + }) +}) diff --git a/apps/web/src/components/settings/roll-api-key-dialog.tsx b/apps/web/src/components/settings/roll-api-key-dialog.tsx index d1ff95845..4040a2c28 100644 --- a/apps/web/src/components/settings/roll-api-key-dialog.tsx +++ b/apps/web/src/components/settings/roll-api-key-dialog.tsx @@ -56,6 +56,11 @@ export function RollApiKeyDialog({ open, onOpenChange, apiKey, onRolled }: RollA onOpenChange(true) return } + // The roll is already dispatched: dismissing now (Escape, outside click, + // the built-in close button) would revoke the old key while the one-time + // replacement secret arrives into a closed dialog — either never seen, or + // shown later under whichever key the dialog opens for next. + if (isRolling) return onOpenChange(false) setNewSecret(null) } diff --git a/apps/web/src/hooks/use-alert-rule-preview.test.ts b/apps/web/src/hooks/use-alert-rule-preview.test.ts new file mode 100644 index 000000000..b74b2d442 --- /dev/null +++ b/apps/web/src/hooks/use-alert-rule-preview.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest" +import { buildRuleCreateParamsV2, defaultRuleForm } from "@/lib/alerts/form-utils" +import { toPreviewForm } from "@/hooks/use-alert-rule-preview" + +describe("toPreviewForm", () => { + it("builds an identical preview payload while non-query fields are being typed", () => { + // The preview atom is keyed by the payload; if name/notes/tag + // edits change it, every keystroke mints a new retained warehouse query. + const base = defaultRuleForm() + const edited = { + ...base, + name: "My critical aler", + notes: "half-typed not", + tags: ["team-a"], + notificationTitle: "It brok", + notificationBody: "Look at ", + } + expect(buildRuleCreateParamsV2(toPreviewForm(edited))).toEqual( + buildRuleCreateParamsV2(toPreviewForm(base)), + ) + }) + + it("still re-keys the payload when a query-relevant field changes", () => { + const base = defaultRuleForm() + const edited = { ...base, threshold: "9" } + expect(buildRuleCreateParamsV2(toPreviewForm(edited))).not.toEqual( + buildRuleCreateParamsV2(toPreviewForm(base)), + ) + }) +}) diff --git a/apps/web/src/hooks/use-alert-rule-preview.ts b/apps/web/src/hooks/use-alert-rule-preview.ts index 719298dd9..777f8237b 100644 --- a/apps/web/src/hooks/use-alert-rule-preview.ts +++ b/apps/web/src/hooks/use-alert-rule-preview.ts @@ -41,6 +41,24 @@ const isPreviewQueryReady = (form: RuleFormState): boolean => { return deriveRuleQueryIssues(form).length === 0 } +/** + * The form with every field the preview query ignores pinned to a constant. + * + * The retained query atom is keyed by the whole payload, so a keystroke in the + * rule's name or notes used to mint a fresh atom — and a fresh warehouse + * request — per character, none of which changed the preview. Exported for the + * regression test. + */ +export const toPreviewForm = (form: RuleFormState): RuleFormState => ({ + ...form, + name: "Untitled rule", + notes: "", + tags: [], + destinationIds: [], + notificationTitle: "", + notificationBody: "", +}) + /** * Evaluator-faithful preview data for the shared alert rule chart * ({@link import("@/components/alerts/alert-rule-chart").AlertRuleChart}). @@ -67,11 +85,7 @@ export function useAlertRulePreview( const payload = useMemo((): V2AlertRulePreviewParams | null => { if (deferredForm === null || !isPreviewQueryReady(deferredForm)) return null - // The rule params require a non-empty name; the preview doesn't care, - // so substitute a placeholder while the user hasn't typed one yet. - const rule = buildRuleCreateParamsV2( - deferredForm.name.trim().length > 0 ? deferredForm : { ...deferredForm, name: "Untitled rule" }, - ) + const rule = buildRuleCreateParamsV2(toPreviewForm(deferredForm)) return { rule, start_time: IsoDateTimeString.make(new Date(normalizeTimestampInput(startTime)).toISOString()), diff --git a/apps/web/src/lib/alerts/chart-series.test.ts b/apps/web/src/lib/alerts/chart-series.test.ts index 72cd21273..9cfda648e 100644 --- a/apps/web/src/lib/alerts/chart-series.test.ts +++ b/apps/web/src/lib/alerts/chart-series.test.ts @@ -1,3 +1,4 @@ +import { Array as Arr } from "effect" import { describe, expect, it } from "vitest" import { AlertRulePreviewPoint, @@ -8,6 +9,7 @@ import { import { clipToDomain, + downsample, GHOST_KEY, mergeGhost, projectPreview, @@ -279,3 +281,46 @@ describe("clipToDomain", () => { ]) }) }) + +describe("downsample", () => { + const rowsOf = (length: number, valueAt: (index: number) => number | null): ChartPoint[] => + Array.from({ length }, (_, index) => ({ t: T0 + index * MINUTE, [SINGLE_KEY]: valueAt(index) })) + + it("returns series within the budget untouched", () => { + const preview = projectPreview( + previewOf([{ groupKey: "all", points: [{ offsetMinutes: 0, value: 1 }] }]), + domain.max, + ) + expect(preview.rows).toHaveLength(1) + }) + + it("keeps a one-window spike that stride sampling used to drop (721 points)", () => { + // 721 points forces downsampling; the spike sits off every stride offset. + const rows = rowsOf(721, (index) => (index === 33 ? 99 : 1)) + const out = downsample(rows) + expect(out.length).toBeLessThanOrEqual(722) + expect(out.some((row) => row[SINGLE_KEY] === 99)).toBe(true) + }) + + it("keeps both the deepest dip and the highest spike at 1441 points", () => { + const rows = rowsOf(1441, (index) => (index === 700 ? -50 : index === 701 ? 120 : 1)) + const out = downsample(rows) + expect(out.some((row) => row[SINGLE_KEY] === -50)).toBe(true) + expect(out.some((row) => row[SINGLE_KEY] === 120)).toBe(true) + // First and last rows always survive, and time order is preserved. + expect(out[0]?.t).toBe(rows[0]?.t) + expect(out[out.length - 1]?.t).toBe(rows[rows.length - 1]?.t) + // Pairing each row with its successor makes the ordering check total. + expect(Arr.zip(out, Arr.drop(out, 1)).every(([previous, next]) => next.t > previous.t)).toBe(true) + }) + + it("keeps a spike carried by a secondary series key", () => { + const rows: ChartPoint[] = Array.from({ length: 900 }, (_, index) => ({ + t: T0 + index * MINUTE, + a: 1, + b: index === 450 ? 77 : 2, + })) + const out = downsample(rows) + expect(out.some((row) => row.b === 77)).toBe(true) + }) +}) diff --git a/apps/web/src/lib/alerts/chart-series.ts b/apps/web/src/lib/alerts/chart-series.ts index 33adc1ed3..229f4194b 100644 --- a/apps/web/src/lib/alerts/chart-series.ts +++ b/apps/web/src/lib/alerts/chart-series.ts @@ -4,6 +4,7 @@ import type { AlertRulePreviewResponse, AlertRulePreviewPoint, } from "@maple/domain/http" +import { Array as Arr, Option } from "effect" import { normalizeTimestampInput } from "@/lib/timezone-format" /** @@ -63,13 +64,61 @@ export interface BucketMeta { // band data stay computed from the full series. const MAX_PLOTTED_POINTS = 720 +/** + * Min/max bucket downsampling: the series is cut into equal buckets and each + * keeps the rows holding its lowest and highest value across every series key, + * in time order, plus the first and last row. A plain stride kept arbitrary + * aligned samples, so a one-window spike or breach — the exact point an alert + * chart exists to show — could vanish once the series exceeded the budget. + */ export function downsample(rows: ReadonlyArray): ChartPoint[] { - if (rows.length <= MAX_PLOTTED_POINTS) return [...rows] - const stride = Math.ceil(rows.length / MAX_PLOTTED_POINTS) + // The guard proves non-emptiness once, so head/last below are total. + if (!Arr.isReadonlyArrayNonEmpty(rows) || rows.length <= MAX_PLOTTED_POINTS) return [...rows] + // Two survivors per bucket keeps the output within the point budget. + const bucketCount = Math.floor(MAX_PLOTTED_POINTS / 2) const out: ChartPoint[] = [] - for (let i = 0; i < rows.length; i += stride) out.push(rows[i]!) - const last = rows[rows.length - 1]! - if (out[out.length - 1] !== last) out.push(last) + const push = (row: ChartPoint) => { + if (out[out.length - 1] !== row) out.push(row) + } + push(Arr.headNonEmpty(rows)) + for (let bucket = 0; bucket < bucketCount; bucket += 1) { + const start = Math.floor((bucket * rows.length) / bucketCount) + const end = Math.floor(((bucket + 1) * rows.length) / bucketCount) + let minRow: Option.Option = Option.none() + let maxRow: Option.Option = Option.none() + let min = Number.POSITIVE_INFINITY + let max = Number.NEGATIVE_INFINITY + for (let i = start; i < end; i += 1) { + const row = rows[i]! + for (const key in row) { + if (key === "t") continue + const value = row[key] + if (typeof value !== "number") continue + if (value < min) { + min = value + minRow = Option.some(row) + } + if (value > max) { + max = value + maxRow = Option.some(row) + } + } + } + if (Option.isNone(minRow) || Option.isNone(maxRow)) { + // Entirely valueless bucket — keep one row so the gap still renders. + if (end > start) Option.match(Arr.get(rows, start), { onNone: () => undefined, onSome: push }) + continue + } + if (minRow.value === maxRow.value) push(minRow.value) + else if (minRow.value.t <= maxRow.value.t) { + push(minRow.value) + push(maxRow.value) + } else { + push(maxRow.value) + push(minRow.value) + } + } + push(Arr.lastNonEmpty(rows)) return out } diff --git a/apps/web/src/lib/alerts/diagnosis.test.ts b/apps/web/src/lib/alerts/diagnosis.test.ts index 0b691b9de..f5cc2906a 100644 --- a/apps/web/src/lib/alerts/diagnosis.test.ts +++ b/apps/web/src/lib/alerts/diagnosis.test.ts @@ -164,3 +164,19 @@ describe("buildDiagnosis", () => { expect(stage(stages, "data").status).toBe("fail") }) }) + +describe("scheduled but never evaluated", () => { + it("warns instead of reporting the rule healthy", () => { + // A worker that claimed the rule and crashed before its first evaluation + // used to read the schedule timestamp as a completed evaluation — passing + // stage, unknown downstream stages, "All stages passing" header. + const stages = diagnose({ + rule: makeRule({ lastEvaluatedAt: null, lastScheduledAt: iso(60_000) }), + states: [], + }) + const evaluated = stage(stages, "evaluated") + expect(evaluated.status).toBe("warn") + expect(evaluated.summary).toContain("no evaluation has completed") + expect(diagnosisVerdict(stages).status).toBe("warn") + }) +}) diff --git a/apps/web/src/lib/alerts/diagnosis.ts b/apps/web/src/lib/alerts/diagnosis.ts index 8d572070f..257f8cc80 100644 --- a/apps/web/src/lib/alerts/diagnosis.ts +++ b/apps/web/src/lib/alerts/diagnosis.ts @@ -104,18 +104,29 @@ export function buildDiagnosis(input: DiagnosisInput): DiagnosisStage[] { summary: "Never evaluated — the scheduler has not picked this rule up yet", evidence: ["New rules are evaluated within about a minute of being enabled."], }) - } else { - const referenceMs = evaluatedAt ?? scheduledAt! - const stale = now - referenceMs > staleThresholdMs(rule) + } else if (evaluatedAt == null && scheduledAt != null) { + // Scheduled is not evaluated. Presenting the schedule timestamp as "last + // evaluated" let a worker that claimed the rule and crashed before its + // first evaluation read as a passing stage — and, with every later stage + // unknown, as "Rule is healthy". + stages.push({ + id: "evaluated", + label: "Evaluated recently", + status: "warn", + summary: `Scheduled ${relative(now, scheduledAt)} but no evaluation has completed yet`, + evidence: [`Last scheduled: ${new Date(scheduledAt).toLocaleString()}`], + }) + } else if (evaluatedAt != null) { + const stale = now - evaluatedAt > staleThresholdMs() stages.push({ id: "evaluated", label: "Evaluated recently", status: stale ? "warn" : "pass", summary: stale - ? `Last evaluated ${relative(now, referenceMs)} — expected roughly every minute` - : `Last evaluated ${relative(now, referenceMs)}`, + ? `Last evaluated ${relative(now, evaluatedAt)} — expected roughly every minute` + : `Last evaluated ${relative(now, evaluatedAt)}`, evidence: [ - evaluatedAt != null ? `Last evaluation: ${new Date(evaluatedAt).toLocaleString()}` : null, + `Last evaluation: ${new Date(evaluatedAt).toLocaleString()}`, scheduledAt != null ? `Last scheduled: ${new Date(scheduledAt).toLocaleString()}` : null, ].filter((line): line is string => line != null), }) diff --git a/apps/web/src/lib/alerts/rule-status.test.ts b/apps/web/src/lib/alerts/rule-status.test.ts index cacc2b402..96acfbb96 100644 --- a/apps/web/src/lib/alerts/rule-status.test.ts +++ b/apps/web/src/lib/alerts/rule-status.test.ts @@ -124,8 +124,8 @@ describe("deriveRuleStatus", () => { expect(result.reason).toBe("Never evaluated") }) - it("marks rules stale past 3x max(window, cadence)", () => { - const rule = makeRule({ lastEvaluatedAt: iso(staleThresholdMs(makeRule()) + 60_000) }) + it("marks rules stale past 3x the state heartbeat", () => { + const rule = makeRule({ lastEvaluatedAt: iso(staleThresholdMs() + 60_000) }) const result = derive({ rule, states: [makeState({ last_evaluated_at: rule.lastEvaluatedAt })], @@ -133,6 +133,30 @@ describe("deriveRuleStatus", () => { expect(result.status).toBe("stale") }) + it("applies the same missed-tick SLA regardless of the rule's window", () => { + // The scheduler evaluates every enabled rule about once a minute; the + // window sizes what an evaluation reads, not how often it runs. Scaling + // staleness by it hid a dead scheduler for 72h on a 24h-window rule. + const gap = staleThresholdMs() + 60_000 + for (const windowMinutes of [5, 60, 1440]) { + const rule = makeRule({ windowMinutes, lastEvaluatedAt: iso(gap) }) + const result = derive({ + rule, + states: [makeState({ last_evaluated_at: rule.lastEvaluatedAt })], + }) + expect(result.status).toBe("stale") + } + // And a fresh evaluation is healthy at every window size. + for (const windowMinutes of [5, 1440]) { + const rule = makeRule({ windowMinutes, lastEvaluatedAt: iso(60_000) }) + const result = derive({ + rule, + states: [makeState({ last_evaluated_at: rule.lastEvaluatedAt })], + }) + expect(result.status).toBe("healthy") + } + }) + it("reports no-data only when every group last skipped", () => { expect(derive({ states: [makeState({ last_status: "skipped" })] }).status).toBe("no-data") expect( diff --git a/apps/web/src/lib/alerts/rule-status.ts b/apps/web/src/lib/alerts/rule-status.ts index e7d8ab46d..ed86a3b72 100644 --- a/apps/web/src/lib/alerts/rule-status.ts +++ b/apps/web/src/lib/alerts/rule-status.ts @@ -22,12 +22,15 @@ export interface DerivedRuleStatus { } /** - * A rule counts as stale when it hasn't been evaluated for 3× the larger of - * its evaluation window and the scheduler cadence (~1 min). Shared with the - * diagnosis panel so both surfaces agree on "stale". + * A rule counts as stale after 3× the server's state heartbeat: the scheduler + * evaluates every enabled rule about once a minute regardless of its window, + * but republishes `last_evaluated_at` at most every ~5 minutes to keep the + * Electric shape quiet. Deliberately independent of `windowMinutes` — the + * window sizes what each evaluation reads, not how often it runs, and scaling + * by it hid scheduler outages for up to 3× the window (72h on a daily rule). + * Shared with the diagnosis panel so both surfaces agree on "stale". */ -export const staleThresholdMs = (rule: Pick): number => - 3 * Math.max(rule.windowMinutes, 5) * 60_000 +export const staleThresholdMs = (): number => 3 * 5 * 60_000 const lastEvaluatedMs = ( rule: AlertRuleDocument, @@ -109,7 +112,7 @@ export function deriveRuleStatus(input: { } const evaluatedAt = lastEvaluatedMs(rule, states) - if (evaluatedAt == null || now - evaluatedAt > staleThresholdMs(rule)) { + if (evaluatedAt == null || now - evaluatedAt > staleThresholdMs()) { return { status: "stale", attention, diff --git a/apps/web/src/lib/alerts/templates.test.ts b/apps/web/src/lib/alerts/templates.test.ts new file mode 100644 index 000000000..32eb48d2f --- /dev/null +++ b/apps/web/src/lib/alerts/templates.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest" +import { defaultRuleForm } from "@/lib/alerts/form-utils" +import { ALERT_TEMPLATES, applyTemplate } from "@/lib/alerts/templates" + +const template = (id: string) => { + const found = ALERT_TEMPLATES.find((t) => t.id === id) + if (!found) throw new Error(`missing template ${id}`) + return found +} + +describe("throughput_drop template", () => { + it("zeroes the minimum sample count so the worst drops still evaluate", () => { + // The evaluator skips windows below the minimum before comparing the + // threshold; with the blank form's 50, a drop to 0–49 samples — the very + // outage the preset exists for — would never breach. + const applied = applyTemplate(template("throughput_drop"), defaultRuleForm()) + expect(applied.minimumSampleCount).toBe("0") + }) +}) + +describe("high_error_rate template", () => { + it("groups by service on an unscoped blank form, matching the MCP template", () => { + const applied = applyTemplate(template("high_error_rate"), defaultRuleForm()) + expect(applied.groupBy).toEqual(["service.name"]) + }) + + it("leaves an explicit service scope alone (scope and grouping are exclusive)", () => { + const applied = applyTemplate(template("high_error_rate"), defaultRuleForm("checkout")) + expect(applied.serviceNames).toEqual(["checkout"]) + expect(applied.groupBy).toEqual([]) + }) + + it("preserves a grouping the user already chose", () => { + const base = { ...defaultRuleForm(), groupBy: ["http.route"] } + const applied = applyTemplate(template("high_error_rate"), base) + expect(applied.groupBy).toEqual(["http.route"]) + }) +}) diff --git a/apps/web/src/lib/alerts/templates.ts b/apps/web/src/lib/alerts/templates.ts index 1e5661225..70640141b 100644 --- a/apps/web/src/lib/alerts/templates.ts +++ b/apps/web/src/lib/alerts/templates.ts @@ -45,6 +45,12 @@ export const ALERT_TEMPLATES: readonly AlertTemplate[] = [ comparator: "gt", threshold: "5", windowMinutes: "5", + // Group per service unless the draft already carries a service scope or + // grouping (they are mutually exclusive server-side). Ungrouped, the rule + // evaluates one org-wide ratio — a small service can fail outright without + // moving a 5% aggregate. Mirrors the MCP template's default. + groupBy: + base.serviceNames.length > 0 || base.groupBy.length > 0 ? base.groupBy : ["service.name"], }), }, { @@ -106,6 +112,11 @@ export const ALERT_TEMPLATES: readonly AlertTemplate[] = [ comparator: "lt", threshold: "100", windowMinutes: "5", + // The evaluator skips windows whose sample count is below the minimum + // BEFORE comparing the threshold, and for throughput the sample count IS + // the signal — the blank form's default of 50 would skip exactly the + // severe drops (0–49 samples), including a total outage. + minimumSampleCount: "0", }), }, ] as const diff --git a/apps/web/src/lib/services/common/auth-headers.test.ts b/apps/web/src/lib/services/common/auth-headers.test.ts index 588f1d680..5e82bd90b 100644 --- a/apps/web/src/lib/services/common/auth-headers.test.ts +++ b/apps/web/src/lib/services/common/auth-headers.test.ts @@ -201,6 +201,59 @@ describe("auth-headers", () => { expect(call).toBe(2) }) + it("fails closed when the org switches while a request is being authorized", async () => { + let release: (() => void) | undefined + const gate = new Promise((resolve) => { + release = resolve + }) + let call = 0 + setMapleAuthHeadersProvider(async () => { + call += 1 + if (call === 1) { + await gate + return { authorization: bearerExpiringIn(600) } + } + return { authorization: bearerExpiringIn(600) } + }) + setActiveOrgId("org_first") + // The request below was constructed under org_first. Re-signing it with + // org_second's bearer would run it — a create, an update — against the + // wrong tenant, so it must be refused, not retried. + const pending = getMapleAuthHeaders() + setActiveOrgId("org_second") + release?.() + + await expect(pending).rejects.toMatchObject({ + _tag: "@maple/web/services/MapleAuthIdentityChangedError", + }) + }) + + it("fails closed when the identity moves again during the boot-path retry", async () => { + // Started with no active org (boot), so one re-resolve is allowed — but + // if the identity changes again mid-retry, the token being returned + // already belongs to an org the user has left. + const gates: Array<() => void> = [] + setMapleAuthHeadersProvider(async () => { + await new Promise((resolve) => { + gates.push(resolve) + }) + return { authorization: bearerExpiringIn(600) } + }) + const pending = getMapleAuthHeaders() + setActiveOrgId("org_b") + // Release the first resolve; the retry starts under org_b's generation. + gates.shift()?.() + await vi.waitFor(() => { + if (gates.length === 0) throw new Error("retry not started") + }) + setActiveOrgId("org_c") + gates.shift()?.() + + await expect(pending).rejects.toMatchObject({ + _tag: "@maple/web/services/MapleAuthIdentityChangedError", + }) + }) + it("drops the cached token on sign-out", async () => { setMapleAuthHeadersProvider(async () => ({ authorization: bearerExpiringIn(600) })) await getMapleAuthHeaders() diff --git a/apps/web/src/lib/services/common/auth-headers.ts b/apps/web/src/lib/services/common/auth-headers.ts index 92f15221c..f1439c84c 100644 --- a/apps/web/src/lib/services/common/auth-headers.ts +++ b/apps/web/src/lib/services/common/auth-headers.ts @@ -1,7 +1,20 @@ import * as Predicate from "effect/Predicate" +import * as Schema from "effect/Schema" export type MapleAuthHeaders = Readonly> +/** + * Raised instead of authorizing a request whose identity changed while its + * headers were resolving. The request was constructed under the org the user + * just left; the API scopes every call by bearer, so re-signing it with the new + * identity's token would run it — including mutations — against the wrong + * tenant. Failing closed lets per-org query state recreate the request itself. + */ +export class MapleAuthIdentityChangedError extends Schema.TaggedError()( + "@maple/web/services/MapleAuthIdentityChangedError", + { message: Schema.String }, +) {} + type MapleAuthHeadersProvider = () => Promise | MapleAuthHeaders let authHeaders: MapleAuthHeaders = {} @@ -156,12 +169,29 @@ const resolveProvidedHeaders = async (): Promise => { export const getMapleAuthHeaders = async (): Promise => { const generation = authGeneration + const orgAtStart = activeOrgId let providedHeaders = await resolveProvidedHeaders() - // The identity changed while we were waiting, so what we are holding belongs - // to an org (or provider) the user has left — the API would resolve it - // against the wrong tenant. Resolve once more under the current identity; - // bounded to one retry so a burst of switches cannot spin here. - if (generation !== authGeneration) providedHeaders = await resolveProvidedHeaders() + if (generation !== authGeneration) { + // The identity changed while we were waiting. If this request was formed + // under a real org, fail closed: its URL and payload belong to the tenant + // the user just left, and re-signing it with the new identity's bearer + // would dispatch it against the wrong org. + if (Predicate.isNotNull(orgAtStart)) { + throw new MapleAuthIdentityChangedError({ + message: "The active organization changed while this request was being authorized.", + }) + } + // Boot path — no previous org, the bump was auth arriving rather than the + // user leaving a tenant. Resolve once more under the current identity, + // and fail closed if it moved again mid-retry. + const retryGeneration = authGeneration + providedHeaders = await resolveProvidedHeaders() + if (retryGeneration !== authGeneration) { + throw new MapleAuthIdentityChangedError({ + message: "The active organization changed while this request was being authorized.", + }) + } + } return { ...providedHeaders, ...authHeaders, diff --git a/lib/clickhouse-builder/src/ch/expr.test-d.ts b/lib/clickhouse-builder/src/ch/expr.test-d.ts index bc80e1799..eeb0a9f51 100644 --- a/lib/clickhouse-builder/src/ch/expr.test-d.ts +++ b/lib/clickhouse-builder/src/ch/expr.test-d.ts @@ -26,6 +26,16 @@ expectTypeOf(CH.min(CH.lit("a"))).toMatchTypeOf>() expectTypeOf(CH.max(CH.lit(1))).toMatchTypeOf>() expectTypeOf(CH.min(CH.lit(1))).toMatchTypeOf>() +// min/max preserve nullability: over a Nullable column ClickHouse returns NULL +// when every contributing value is NULL, so the result must not narrow to the +// non-null type. +declare const nullableNum: Expr +expectTypeOf(CH.min(nullableNum)).toEqualTypeOf>() +expectTypeOf(CH.max(nullableNum)).toEqualTypeOf>() + +// ifNull with a non-nullable fallback strips the null the aggregate kept +expectTypeOf(CH.ifNull(nullableNum, CH.lit(0))).toEqualTypeOf>() + // any_ preserves generic expectTypeOf(CH.any(CH.lit("x"))).toMatchTypeOf>() expectTypeOf(CH.any(CH.lit(1))).toMatchTypeOf>() diff --git a/lib/clickhouse-builder/src/ch/functions/aggregate.ts b/lib/clickhouse-builder/src/ch/functions/aggregate.ts index d4defef51..341a254a5 100644 --- a/lib/clickhouse-builder/src/ch/functions/aggregate.ts +++ b/lib/clickhouse-builder/src/ch/functions/aggregate.ts @@ -32,12 +32,15 @@ export const minIf = defineFn<[Expr, Condition], number>("minIf", T.floa // These hand back one of their arguments unchanged, so they decode as it does. // `sameAs(0)` is that rule by name — each of them used to carry its own copy. +// +// min/max over a Nullable column stay nullable: ClickHouse skips NULLs but +// returns NULL when every contributing value is NULL, so stripping the `| null` +// here (as an earlier version did with `NonNullable`) lied to every caller +// while `sameAs(0)` kept the nullable runtime codec. -export const min_ = (expr: Expr): Expr> => - defineFn<[Expr], NonNullable>("min", sameAs(0))(expr) +export const min_ = (expr: Expr): Expr => defineFn<[Expr], T>("min", sameAs(0))(expr) -export const max_ = (expr: Expr): Expr> => - defineFn<[Expr], NonNullable>("max", sameAs(0))(expr) +export const max_ = (expr: Expr): Expr => defineFn<[Expr], T>("max", sameAs(0))(expr) export const any_ = (expr: Expr): Expr => defineFn<[Expr], T>("any", sameAs(0))(expr) diff --git a/lib/clickhouse-builder/src/ch/functions/conditional.ts b/lib/clickhouse-builder/src/ch/functions/conditional.ts index fb38f8398..10f279fc5 100644 --- a/lib/clickhouse-builder/src/ch/functions/conditional.ts +++ b/lib/clickhouse-builder/src/ch/functions/conditional.ts @@ -35,6 +35,14 @@ export function multiIf(cases: Array<[Condition, Expr]>, else_: Expr): export const coalesce = (...exprs: Expr[]): Expr => defineFn[], T>("coalesce", firstTypedNonNull())(...exprs) +/** + * `ifNull(expr, fallback)` — `expr` unless it is NULL, else `fallback`. The + * two-argument coalesce, typed so a non-nullable fallback strips the `| null` + * that {@link coalesce}'s single `T` cannot. + */ +export const ifNull = (expr: Expr, fallback: Expr): Expr => + defineFn<[Expr, Expr], T>("ifNull", firstTypedNonNull())(expr, fallback) + export function nullIf(expr: Expr, value: Expr | string): Expr { // The result is `expr` or NULL, so it decodes as `expr` does — nullably. const schema = schemaOf(expr) diff --git a/lib/clickhouse-builder/src/ch/functions/index.ts b/lib/clickhouse-builder/src/ch/functions/index.ts index 2841e375d..700d25d56 100644 --- a/lib/clickhouse-builder/src/ch/functions/index.ts +++ b/lib/clickhouse-builder/src/ch/functions/index.ts @@ -75,7 +75,7 @@ export { toDateTime, } from "./date-time" -export { if_, multiIf, coalesce, nullIf, ifNotFinite } from "./conditional" +export { if_, multiIf, coalesce, ifNull, nullIf, ifNotFinite } from "./conditional" export { arrayDistinct, diff --git a/lib/clickhouse-builder/src/ch/index.ts b/lib/clickhouse-builder/src/ch/index.ts index b26946dc4..815a71264 100644 --- a/lib/clickhouse-builder/src/ch/index.ts +++ b/lib/clickhouse-builder/src/ch/index.ts @@ -184,6 +184,7 @@ export { if_, multiIf, coalesce, + ifNull, nullIf, ifNotFinite, // Array diff --git a/packages/browser/src/tracing.test.ts b/packages/browser/src/tracing.test.ts index e71edbd4a..0c577d421 100644 --- a/packages/browser/src/tracing.test.ts +++ b/packages/browser/src/tracing.test.ts @@ -1,6 +1,12 @@ // @vitest-environment jsdom // TEST-SEAM: This focused test replaces process-global modules that have no instance-level injection seam. -import { trace } from "@opentelemetry/api" +import { + INVALID_SPAN_CONTEXT, + type Span as ApiSpan, + trace, + type Tracer, + type TracerProvider, +} from "@opentelemetry/api" import type { ReadableSpan, Span } from "@opentelemetry/sdk-trace-base" import { afterEach, describe, expect, it, vi } from "vitest" @@ -127,6 +133,40 @@ describe("setupTracing unload flush", () => { expect(exported).toHaveLength(0) }) + it("exports spans again after init → shutdown → init, without a manual trace.disable", async () => { + // The global OTel registration is first-write-wins: unless shutdown + // releases it, the proxy keeps delegating to the shut-down provider and a + // second SDK session silently exports nothing. + const first = setupTracing(CONFIG) + endOneSpan() + window.dispatchEvent(new Event("pagehide")) + await vi.waitFor(() => expect(exported).toHaveLength(1)) + await first() + + shutdown = setupTracing(CONFIG) + endOneSpan() + window.dispatchEvent(new Event("pagehide")) + await vi.waitFor(() => expect(exported).toHaveLength(2)) + }) + + it("leaves a host app's earlier provider registration alone on shutdown", async () => { + // A host that registered its own provider owns the globals; losing them + // (trace.disable) would break the host's tracing, not just ours. + const hostSpan: ApiSpan = trace.wrapSpanContext(INVALID_SPAN_CONTEXT) + const hostTracer: Tracer = { + startSpan: vi.fn(() => hostSpan), + startActiveSpan: vi.fn(), + } + const hostProvider: TracerProvider = { getTracer: () => hostTracer } + trace.setGlobalTracerProvider(hostProvider) + + const teardown = setupTracing(CONFIG) + await teardown() + + expect(trace.getTracer("host").startSpan("still-host")).toBeDefined() + expect(hostTracer.startSpan).toHaveBeenCalledWith("still-host") + }) + it("removes its listeners on shutdown", async () => { const teardown = setupTracing(CONFIG) await teardown() diff --git a/packages/browser/src/tracing.ts b/packages/browser/src/tracing.ts index 2e928a873..31df24db4 100644 --- a/packages/browser/src/tracing.ts +++ b/packages/browser/src/tracing.ts @@ -6,6 +6,7 @@ import { SDK_HINT_HEADER, sdkHint, } from "@maple/browser-session" +import { context, propagation, ProxyTracerProvider, trace } from "@opentelemetry/api" import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http" import { registerInstrumentations } from "@opentelemetry/instrumentation" import { FetchInstrumentation } from "@opentelemetry/instrumentation-fetch" @@ -138,6 +139,13 @@ export function setupTracing(config: ResolvedConfig): () => Promise { ], }) provider.register() + // The OTel globals are first-write-wins. If a host app registered its own + // provider before us, ours never became the global one — remember whether we + // won so shutdown releases only globals this SDK actually owns. + const globalProvider = trace.getTracerProvider() + const ownsGlobals = + globalProvider === provider || + (globalProvider instanceof ProxyTracerProvider && globalProvider.getDelegate() === provider) // Without this the batch processor's queue dies with the tab: its only flush // is `provider.shutdown()`, which a host app that never calls `shutdown()` @@ -182,6 +190,15 @@ export function setupTracing(config: ResolvedConfig): () => Promise { } unregisterInstrumentations?.() await provider.shutdown() + // Release the globals so a later init() can register a live provider. + // The global proxy keeps delegating to this now-shut-down provider + // otherwise, and the documented init → shutdown → init lifecycle would + // silently export nothing on the second session. + if (ownsGlobals) { + trace.disable() + context.disable() + propagation.disable() + } } } diff --git a/packages/db/drizzle/0054_alert_incident_open_uniqueness.sql b/packages/db/drizzle/0054_alert_incident_open_uniqueness.sql new file mode 100644 index 000000000..3e019303b --- /dev/null +++ b/packages/db/drizzle/0054_alert_incident_open_uniqueness.sql @@ -0,0 +1,23 @@ +-- Duplicate open incidents predate these constraints (expired scheduler/org +-- claims let two ticks open the same incident); resolve all but the freshest +-- per key first or the CREATEs fail validation. +WITH ranked AS ( + SELECT id, row_number() OVER ( + PARTITION BY org_id, rule_id, group_key + ORDER BY last_triggered_at DESC, id DESC + ) AS rn + FROM alert_incidents WHERE status = 'open' AND group_key IS NOT NULL +) +UPDATE alert_incidents SET status = 'resolved', resolved_at = now(), updated_at = now() +WHERE id IN (SELECT id FROM ranked WHERE rn > 1);--> statement-breakpoint +WITH ranked AS ( + SELECT id, row_number() OVER ( + PARTITION BY org_id, detector_key + ORDER BY last_triggered_at DESC, id DESC + ) AS rn + FROM anomaly_incidents WHERE status = 'open' +) +UPDATE anomaly_incidents SET status = 'resolved', resolved_at = now(), updated_at = now() +WHERE id IN (SELECT id FROM ranked WHERE rn > 1);--> statement-breakpoint +CREATE UNIQUE INDEX "alert_incidents_open_group_idx" ON "alert_incidents" USING btree ("org_id","rule_id","group_key") WHERE "alert_incidents"."status" = 'open';--> statement-breakpoint +CREATE UNIQUE INDEX "anomaly_incidents_open_detector_idx" ON "anomaly_incidents" USING btree ("org_id","detector_key") WHERE "anomaly_incidents"."status" = 'open'; diff --git a/packages/db/drizzle/meta/0054_snapshot.json b/packages/db/drizzle/meta/0054_snapshot.json new file mode 100644 index 000000000..387eae2cc --- /dev/null +++ b/packages/db/drizzle/meta/0054_snapshot.json @@ -0,0 +1,8823 @@ +{ + "id": "08d43f4b-2c3a-4173-8e8e-1999ae66b5d0", + "prevId": "17544f97-9468-41aa-96a9-54260c5aad29", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.ai_triage_settings": { + "name": "ai_triage_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "max_runs_per_day": { + "name": "max_runs_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20 + }, + "max_passes_per_day": { + "name": "max_passes_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 90 + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_delivery_events": { + "name": "alert_delivery_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt_number": { + "name": "attempt_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_message": { + "name": "provider_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_reference": { + "name": "provider_reference", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_delivery_events_org_idx": { + "name": "alert_delivery_events_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_org_incident_idx": { + "name": "alert_delivery_events_org_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_due_idx": { + "name": "alert_delivery_events_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_claim_idx": { + "name": "alert_delivery_events_claim_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_delivery_attempt_idx": { + "name": "alert_delivery_events_delivery_attempt_idx", + "columns": [ + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_destinations": { + "name": "alert_destinations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "config_json": { + "name": "config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_tested_at": { + "name": "last_tested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_error": { + "name": "last_test_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_failure_at": { + "name": "last_failure_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_reason": { + "name": "disabled_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_destinations_org_idx": { + "name": "alert_destinations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_enabled_idx": { + "name": "alert_destinations_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_name_idx": { + "name": "alert_destinations_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_incidents": { + "name": "alert_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_key": { + "name": "incident_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_name": { + "name": "rule_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_delivered_event_type": { + "name": "last_delivered_event_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_notified_at": { + "name": "last_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_incidents_org_idx": { + "name": "alert_incidents_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_status_idx": { + "name": "alert_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_rule_idx": { + "name": "alert_incidents_org_rule_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "rule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_issue_idx": { + "name": "alert_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_incident_key_idx": { + "name": "alert_incidents_incident_key_idx", + "columns": [ + { + "expression": "incident_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_open_group_idx": { + "name": "alert_incidents_open_group_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "rule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "group_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"alert_incidents\".\"status\" = 'open'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rule_claims": { + "name": "alert_rule_claims", + "schema": "", + "columns": { + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rule_claims_org_idx": { + "name": "alert_rule_claims_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rule_states": { + "name": "alert_rule_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'__total__'" + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rule_states_org_idx": { + "name": "alert_rule_states_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "alert_rule_states_org_id_rule_id_group_key_pk": { + "name": "alert_rule_states_org_id_rule_id_group_key_pk", + "columns": [ + "org_id", + "rule_id", + "group_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rules": { + "name": "alert_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_template_json": { + "name": "notification_template_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_names_json": { + "name": "service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "exclude_service_names_json": { + "name": "exclude_service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "environments_json": { + "name": "environments_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tags_json": { + "name": "tags_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "window_minutes": { + "name": "window_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minimum_sample_count": { + "name": "minimum_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_breaches_required": { + "name": "consecutive_breaches_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "consecutive_healthy_required": { + "name": "consecutive_healthy_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "renotify_interval_minutes": { + "name": "renotify_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 30 + }, + "apdex_threshold_ms": { + "name": "apdex_threshold_ms", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "query_builder_draft_json": { + "name": "query_builder_draft_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "raw_query_sql": { + "name": "raw_query_sql", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "group_by": { + "name": "group_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "query_spec_json": { + "name": "query_spec_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "reducer": { + "name": "reducer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sample_count_strategy": { + "name": "sample_count_strategy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "no_data_behavior": { + "name": "no_data_behavior", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rules_org_idx": { + "name": "alert_rules_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_enabled_idx": { + "name": "alert_rules_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_name_idx": { + "name": "alert_rules_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_settings": { + "name": "anomaly_detector_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sensitivity": { + "name": "sensitivity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'normal'" + }, + "muted_signals_json": { + "name": "muted_signals_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_tick_at": { + "name": "last_tick_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_states": { + "name": "anomaly_detector_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_incident_id": { + "name": "last_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_detector_states_open_incident_idx": { + "name": "anomaly_detector_states_open_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "open_incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"anomaly_detector_states\".\"open_incident_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_detector_states_evaluated_idx": { + "name": "anomaly_detector_states_evaluated_idx", + "columns": [ + { + "expression": "last_evaluated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "anomaly_detector_states_org_id_detector_key_pk": { + "name": "anomaly_detector_states_org_id_detector_key_pk", + "columns": [ + "org_id", + "detector_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_incidents": { + "name": "anomaly_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "opened_value": { + "name": "opened_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_sigma": { + "name": "baseline_sigma", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_value": { + "name": "threshold_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolve_reason": { + "name": "resolve_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "triage_status": { + "name": "triage_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprints_json": { + "name": "fingerprints_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "reopen_count": { + "name": "reopen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_reopened_at": { + "name": "last_reopened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_incidents_org_status_triggered_idx": { + "name": "anomaly_incidents_org_status_triggered_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_triggered_idx": { + "name": "anomaly_incidents_org_triggered_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_detector_idx": { + "name": "anomaly_incidents_org_detector_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "detector_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_issue_idx": { + "name": "anomaly_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_open_detector_idx": { + "name": "anomaly_incidents_open_detector_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "detector_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"anomaly_incidents\".\"status\" = 'open'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_prefix": { + "name": "key_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'standard'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by_email": { + "name": "created_by_email", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_keys_org_id_idx": { + "name": "api_keys_org_id_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_analytics_state": { + "name": "cloudflare_analytics_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "watermark_at": { + "name": "watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_at": { + "name": "backfill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "settings_fetched_at": { + "name": "settings_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "quantiles_available": { + "name": "quantiles_available", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovered_at": { + "name": "discovered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "live_scripts_json": { + "name": "live_scripts_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cf_analytics_state_org_account_dataset_zone_idx": { + "name": "cf_analytics_state_org_account_dataset_zone_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dataset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "zone_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cf_analytics_state_org_idx": { + "name": "cf_analytics_state_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_hyperdrive_configs": { + "name": "cloudflare_hyperdrive_configs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_host": { + "name": "origin_host", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_port": { + "name": "origin_port", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "origin_scheme": { + "name": "origin_scheme", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_database": { + "name": "origin_database", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_user": { + "name": "origin_user", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cloudflare_hyperdrive_configs_org_config_idx": { + "name": "cloudflare_hyperdrive_configs_org_config_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "config_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_hyperdrive_configs_org_idx": { + "name": "cloudflare_hyperdrive_configs_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_logpush_connectors": { + "name": "cloudflare_logpush_connectors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'http_requests'" + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_received_at": { + "name": "last_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_rotated_at": { + "name": "secret_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cloudflare_logpush_connectors_org_idx": { + "name": "cloudflare_logpush_connectors_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_org_enabled_idx": { + "name": "cloudflare_logpush_connectors_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_secret_hash_unique": { + "name": "cloudflare_logpush_connectors_secret_hash_unique", + "columns": [ + { + "expression": "secret_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cli_device_authorizations": { + "name": "cli_device_authorizations", + "schema": "", + "columns": { + "device_code_hash": { + "name": "device_code_hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_code_hash": { + "name": "user_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_name": { + "name": "device_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_roles": { + "name": "approved_roles", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "approved_user_email": { + "name": "approved_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_id": { + "name": "api_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "denied_at": { + "name": "denied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cli_device_authorizations_user_code_unique": { + "name": "cli_device_authorizations_user_code_unique", + "columns": [ + { + "expression": "user_code_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_device_authorizations_expires_idx": { + "name": "cli_device_authorizations_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_authorizations": { + "name": "mcp_oauth_authorizations", + "schema": "", + "columns": { + "request_id_hash": { + "name": "request_id_hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_code_hash": { + "name": "authorization_code_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_roles": { + "name": "approved_roles", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "approved_user_email": { + "name": "approved_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "denied_at": { + "name": "denied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "used_at": { + "name": "used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_oauth_authorizations_code_unique": { + "name": "mcp_oauth_authorizations_code_unique", + "columns": [ + { + "expression": "authorization_code_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_authorizations_expires_idx": { + "name": "mcp_oauth_authorizations_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_clients": { + "name": "mcp_oauth_clients", + "schema": "", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "client_uri": { + "name": "client_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_refresh_tokens": { + "name": "mcp_oauth_refresh_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "roles": { + "name": "roles", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_key_id": { + "name": "access_key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replaced_by_id": { + "name": "replaced_by_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "family_expires_at": { + "name": "family_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mcp_oauth_refresh_tokens_hash_unique": { + "name": "mcp_oauth_refresh_tokens_hash_unique", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_refresh_tokens_family_idx": { + "name": "mcp_oauth_refresh_tokens_family_idx", + "columns": [ + { + "expression": "family_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_refresh_tokens_expires_idx": { + "name": "mcp_oauth_refresh_tokens_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_devices": { + "name": "mobile_devices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bundle_id": { + "name": "bundle_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_version": { + "name": "app_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "live_activity_start_token": { + "name": "live_activity_start_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_name": { + "name": "device_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preferences": { + "name": "preferences", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "disabled_at": { + "name": "disabled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_reason": { + "name": "disabled_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_pushed_at": { + "name": "last_pushed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mobile_devices_org_platform_token_unique": { + "name": "mobile_devices_org_platform_token_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "platform", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mobile_devices_org_idx": { + "name": "mobile_devices_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mobile_devices_user_idx": { + "name": "mobile_devices_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboard_shares": { + "name": "dashboard_shares", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "widget_id": { + "name": "widget_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_suffix": { + "name": "token_suffix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "dashboard_shares_token_hash_unq": { + "name": "dashboard_shares_token_hash_unq", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_live_unq": { + "name": "dashboard_shares_live_unq", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(widget_id, '')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "revoked_at is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_org_dashboard_idx": { + "name": "dashboard_shares_org_dashboard_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_id_idx": { + "name": "dashboard_shares_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "dashboard_shares_dashboard_fk": { + "name": "dashboard_shares_dashboard_fk", + "tableFrom": "dashboard_shares", + "tableTo": "dashboards", + "columnsFrom": [ + "org_id", + "dashboard_id" + ], + "columnsTo": [ + "org_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "dashboard_shares_org_id_id_pk": { + "name": "dashboard_shares_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboard_versions": { + "name": "dashboard_versions", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version_number": { + "name": "version_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "change_kind": { + "name": "change_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_version_id": { + "name": "source_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "dashboard_versions_org_dashboard_idx": { + "name": "dashboard_versions_org_dashboard_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_versions_org_dashboard_version_unq": { + "name": "dashboard_versions_org_dashboard_version_unq", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboard_versions_org_id_id_pk": { + "name": "dashboard_versions_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboards": { + "name": "dashboards", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "dashboards_org_updated_idx": { + "name": "dashboards_org_updated_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboards_org_name_idx": { + "name": "dashboards_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboards_org_id_id_pk": { + "name": "dashboards_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.digest_subscriptions": { + "name": "digest_subscriptions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "opted_out_at": { + "name": "opted_out_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "day_of_week": { + "name": "day_of_week", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "namespaces_json": { + "name": "namespaces_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "environments_json": { + "name": "environments_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'[]'" + }, + "last_sent_at": { + "name": "last_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_attempted_at": { + "name": "last_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "digest_subscriptions_org_user_idx": { + "name": "digest_subscriptions_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "digest_subscriptions_org_enabled_idx": { + "name": "digest_subscriptions_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.actors": { + "name": "actors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agent_name": { + "name": "agent_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capabilities_json": { + "name": "capabilities_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "actors_org_user_idx": { + "name": "actors_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_agent_name_idx": { + "name": "actors_org_agent_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "agent_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_type_idx": { + "name": "actors_org_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_fingerprint_candidates": { + "name": "error_fingerprint_candidates", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_versions_json": { + "name": "service_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_fingerprint_candidates_last_seen_idx": { + "name": "error_fingerprint_candidates_last_seen_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_fingerprint_candidates_org_id_fingerprint_hash_pk": { + "name": "error_fingerprint_candidates_org_id_fingerprint_hash_pk", + "columns": [ + "org_id", + "fingerprint_hash" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_incidents": { + "name": "error_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_incidents_org_issue_idx": { + "name": "error_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_incidents_org_status_idx": { + "name": "error_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_events": { + "name": "error_issue_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_state": { + "name": "from_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "to_state": { + "name": "to_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_events_issue_idx": { + "name": "error_issue_events_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_actor_idx": { + "name": "error_issue_events_actor_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_type_idx": { + "name": "error_issue_events_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_pull_requests": { + "name": "error_issue_pull_requests", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repo_full_name": { + "name": "repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "merge_commit_sha": { + "name": "merge_commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "link_source": { + "name": "link_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "linked_by_actor_id": { + "name": "linked_by_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_pull_requests_issue_pr_idx": { + "name": "error_issue_pull_requests_issue_pr_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo_full_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_pull_requests_repo_number_idx": { + "name": "error_issue_pull_requests_repo_number_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo_full_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_pull_requests_issue_idx": { + "name": "error_issue_pull_requests_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_states": { + "name": "error_issue_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_observed_occurrence_at": { + "name": "last_observed_occurrence_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_issue_states_org_id_issue_id_pk": { + "name": "error_issue_states_org_id_issue_id_pk", + "columns": [ + "org_id", + "issue_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_verifications": { + "name": "error_issue_verifications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pull_request_id": { + "name": "pull_request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'waiting'" + }, + "merged_at": { + "name": "merged_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "verify_after": { + "name": "verify_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "baseline_versions_json": { + "name": "baseline_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "baseline_occurrence_count": { + "name": "baseline_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "baseline_rate_per_hour": { + "name": "baseline_rate_per_hour", + "type": "double precision", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "verdict": { + "name": "verdict", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "verdict_note": { + "name": "verdict_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "post_merge_occurrence_count": { + "name": "post_merge_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_verifications_due_idx": { + "name": "error_issue_verifications_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "verify_after", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_verifications_issue_idx": { + "name": "error_issue_verifications_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_verifications_open_idx": { + "name": "error_issue_verifications_open_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"error_issue_verifications\".\"status\" in ('waiting', 'running')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issues": { + "name": "error_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'error'" + }, + "source_ref_json": { + "name": "source_ref_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint_version": { + "name": "fingerprint_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_state": { + "name": "workflow_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'triage'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "severity_source": { + "name": "severity_source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_actor_id": { + "name": "assigned_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_holder_actor_id": { + "name": "lease_holder_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolved_by_actor_id": { + "name": "resolved_by_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_regressed_at": { + "name": "last_regressed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "regression_count": { + "name": "regression_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "seen_versions_json": { + "name": "seen_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "resolved_versions_json": { + "name": "resolved_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "snooze_until": { + "name": "snooze_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issues_org_fp_idx": { + "name": "error_issues_org_fp_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_workflow_idx": { + "name": "error_issues_org_workflow_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_severity_idx": { + "name": "error_issues_org_severity_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "severity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_live_seen_idx": { + "name": "error_issues_org_live_seen_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"error_issues\".\"archived_at\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_fp_version_idx": { + "name": "error_issues_org_fp_version_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_assignee_idx": { + "name": "error_issues_org_assignee_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "assigned_actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_lease_expiry_idx": { + "name": "error_issues_lease_expiry_idx", + "columns": [ + { + "expression": "lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_archived_idx": { + "name": "error_issues_org_archived_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"error_issues\".\"archived_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_notification_deliveries": { + "name": "error_notification_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_notification_deliveries_due_idx": { + "name": "error_notification_deliveries_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_notification_deliveries_org_idx": { + "name": "error_notification_deliveries_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_notification_deliveries_key_destination_idx": { + "name": "error_notification_deliveries_key_destination_idx", + "columns": [ + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_notification_policies": { + "name": "error_notification_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "notify_on_first_seen": { + "name": "notify_on_first_seen", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_regression": { + "name": "notify_on_regression", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_resolve": { + "name": "notify_on_resolve", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_in_review": { + "name": "notify_on_transition_in_review", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_done": { + "name": "notify_on_transition_done", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_claim": { + "name": "notify_on_claim", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "min_occurrence_count": { + "name": "min_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'warning'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_tick_states": { + "name": "error_tick_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "processed_through": { + "name": "processed_through", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "bootstrap_completed": { + "name": "bootstrap_completed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "claim_token": { + "name": "claim_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_tick_states_claim_idx": { + "name": "error_tick_states_claim_idx", + "columns": [ + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalation_policies": { + "name": "issue_escalation_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "rules_json": { + "name": "rules_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalations": { + "name": "issue_escalations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "delivery_results_json": { + "name": "delivery_results_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "issue_escalations_dedupe_idx": { + "name": "issue_escalations_dedupe_idx", + "columns": [ + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_due_idx": { + "name": "issue_escalations_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_org_issue_idx": { + "name": "issue_escalations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.investigation_lens_runs": { + "name": "investigation_lens_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "lens_id": { + "name": "lens_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "verdict": { + "name": "verdict", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "claim": { + "name": "claim", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress_note": { + "name": "progress_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "elapsed_ms": { + "name": "elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "lens_name": { + "name": "lens_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lens_question": { + "name": "lens_question", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deadline_hit": { + "name": "deadline_hit", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "hypothesis_json": { + "name": "hypothesis_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_json": { + "name": "evidence_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mechanism": { + "name": "mechanism", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "self_doubt": { + "name": "self_doubt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "suggested_actions_json": { + "name": "suggested_actions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reported_at": { + "name": "reported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ranked_at": { + "name": "ranked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "investigation_lens_runs_lens_idx": { + "name": "investigation_lens_runs_lens_idx", + "columns": [ + { + "expression": "investigation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lens_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigation_lens_runs_org_inv_idx": { + "name": "investigation_lens_runs_org_inv_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "investigation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "investigation_lens_runs_investigation_id_investigations_id_fk": { + "name": "investigation_lens_runs_investigation_id_investigations_id_fk", + "tableFrom": "investigation_lens_runs", + "tableTo": "investigations", + "columnsFrom": [ + "investigation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.investigations": { + "name": "investigations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'investigating'" + }, + "seeded_by": { + "name": "seeded_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user'" + }, + "subject_json": { + "name": "subject_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "incident_kind": { + "name": "incident_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "report_json": { + "name": "report_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fanout_state": { + "name": "fanout_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "fanout_size": { + "name": "fanout_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "plan_json": { + "name": "plan_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "planner_model": { + "name": "planner_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "planner_elapsed_ms": { + "name": "planner_elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "validator_note": { + "name": "validator_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validator_elapsed_ms": { + "name": "validator_elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "fanout_deadline_at": { + "name": "fanout_deadline_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fanout_attempt": { + "name": "fanout_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "autonomous_turns": { + "name": "autonomous_turns", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "diagnosed_at": { + "name": "diagnosed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "investigations_incident_idx": { + "name": "investigations_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"investigations\".\"incident_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_created_idx": { + "name": "investigations_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_issue_idx": { + "name": "investigations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_status_idx": { + "name": "investigations_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.live_activities": { + "name": "live_activities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_id": { + "name": "device_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activity_id": { + "name": "activity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "push_token": { + "name": "push_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ended_reason": { + "name": "ended_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "live_activities_device_incident_unique": { + "name": "live_activities_device_incident_unique", + "columns": [ + { + "expression": "device_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "live_activities_incident_idx": { + "name": "live_activities_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_auth_states": { + "name": "oauth_auth_states", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "initiated_by_user_id": { + "name": "initiated_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "return_to": { + "name": "return_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_auth_states_expires_idx": { + "name": "oauth_auth_states_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_connections": { + "name": "oauth_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_id": { + "name": "external_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_email": { + "name": "external_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_account_name": { + "name": "external_account_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "granted_accounts_json": { + "name": "granted_accounts_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "access_token_ciphertext": { + "name": "access_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_iv": { + "name": "access_token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_tag": { + "name": "access_token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token_ciphertext": { + "name": "refresh_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_iv": { + "name": "refresh_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_tag": { + "name": "refresh_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_connections_org_provider_idx": { + "name": "oauth_connections_org_provider_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_connections_org_idx": { + "name": "oauth_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_onboarding_state": { + "name": "org_onboarding_state", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_data_requested": { + "name": "demo_data_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "checklist_dismissed_at": { + "name": "checklist_dismissed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "first_data_received_at": { + "name": "first_data_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "welcome_email_sent_at": { + "name": "welcome_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "connect_nudge_email_sent_at": { + "name": "connect_nudge_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "stalled_email_sent_at": { + "name": "stalled_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "activation_email_sent_at": { + "name": "activation_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_attribute_mappings": { + "name": "org_ingest_attribute_mappings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_context": { + "name": "source_context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_key": { + "name": "target_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_ingest_attribute_mappings_org_idx": { + "name": "org_ingest_attribute_mappings_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_recommendation_issues": { + "name": "org_recommendation_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "recommendation_key": { + "name": "recommendation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_key": { + "name": "canonical_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "usage_count": { + "name": "usage_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "org_recommendation_issues_org_idx": { + "name": "org_recommendation_issues_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_recommendation_issues_org_key_idx": { + "name": "org_recommendation_issues_org_key_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "recommendation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_keys": { + "name": "org_ingest_keys", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key_hash": { + "name": "public_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_ciphertext": { + "name": "private_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_iv": { + "name": "private_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_tag": { + "name": "private_key_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_hash": { + "name": "private_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_rotated_at": { + "name": "public_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "private_rotated_at": { + "name": "private_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "org_ingest_keys_public_key_unique": { + "name": "org_ingest_keys_public_key_unique", + "columns": [ + { + "expression": "public_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_public_key_hash_unique": { + "name": "org_ingest_keys_public_key_hash_unique", + "columns": [ + { + "expression": "public_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_private_key_hash_unique": { + "name": "org_ingest_keys_private_key_hash_unique", + "columns": [ + { + "expression": "private_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_ingest_keys_org_id_pk": { + "name": "org_ingest_keys_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_sampling_policies": { + "name": "org_ingest_sampling_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "trace_sample_ratio": { + "name": "trace_sample_ratio", + "type": "double precision", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "always_keep_error_spans": { + "name": "always_keep_error_spans", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "always_keep_slow_spans_ms": { + "name": "always_keep_slow_spans_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_settings": { + "name": "org_clickhouse_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_url": { + "name": "ch_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_user": { + "name": "ch_user", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_password_ciphertext": { + "name": "ch_password_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_iv": { + "name": "ch_password_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_tag": { + "name": "ch_password_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_database": { + "name": "ch_database", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema_version": { + "name": "schema_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_settings_org_id_pk": { + "name": "org_clickhouse_settings_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_schema_apply_runs": { + "name": "org_clickhouse_schema_apply_runs", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_migration": { + "name": "current_migration", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_total": { + "name": "steps_total", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_done": { + "name": "steps_done", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "applied_versions": { + "name": "applied_versions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "skipped": { + "name": "skipped", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_schema_apply_runs_org_id_pk": { + "name": "org_clickhouse_schema_apply_runs_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_connections": { + "name": "planetscale_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ps_organization": { + "name": "ps_organization", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scrape_target_id": { + "name": "scrape_target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_ciphertext": { + "name": "webhook_secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_iv": { + "name": "webhook_secret_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_tag": { + "name": "webhook_secret_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detected_permissions_json": { + "name": "detected_permissions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_inventory_at": { + "name": "last_inventory_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_inventory_error": { + "name": "last_inventory_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_connections_org_idx": { + "name": "planetscale_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_databases": { + "name": "planetscale_databases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mysql'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "branches_json": { + "name": "branches_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_databases_org_db_idx": { + "name": "planetscale_databases_org_db_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_databases_org_idx": { + "name": "planetscale_databases_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_events": { + "name": "planetscale_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "database_name": { + "name": "database_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch_name": { + "name": "branch_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_login": { + "name": "actor_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_events_dedupe_idx": { + "name": "planetscale_events_dedupe_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_events_org_db_time_idx": { + "name": "planetscale_events_org_db_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_events_org_time_idx": { + "name": "planetscale_events_org_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_poll_state": { + "name": "planetscale_poll_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "watermark_at": { + "name": "watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_poll_state_org_dataset_db_idx": { + "name": "planetscale_poll_state_org_dataset_db_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dataset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_poll_state_org_idx": { + "name": "planetscale_poll_state_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_target_checks": { + "name": "scrape_target_checks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "byDefault", + "name": "scrape_target_checks_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_target_key": { + "name": "sub_target_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_scraped": { + "name": "samples_scraped", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_post_relabel": { + "name": "samples_post_relabel", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "scrape_target_checks_target_checked_idx": { + "name": "scrape_target_checks_target_checked_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scrape_target_checks_target_id_scrape_targets_id_fk": { + "name": "scrape_target_checks_target_id_scrape_targets_id_fk", + "tableFrom": "scrape_target_checks", + "tableTo": "scrape_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_targets": { + "name": "scrape_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'prometheus'" + }, + "discovery_config_json": { + "name": "discovery_config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scrape_interval_seconds": { + "name": "scrape_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 15 + }, + "labels_json": { + "name": "labels_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "managed_by": { + "name": "managed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_ciphertext": { + "name": "auth_credentials_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_iv": { + "name": "auth_credentials_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_tag": { + "name": "auth_credentials_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_scrape_at": { + "name": "last_scrape_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_scrape_error": { + "name": "last_scrape_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "scrape_targets_org_idx": { + "name": "scrape_targets_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scrape_targets_org_enabled_idx": { + "name": "scrape_targets_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_workspaces": { + "name": "slack_workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_id": { + "name": "team_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_name": { + "name": "team_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_user_id": { + "name": "bot_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_ciphertext": { + "name": "bot_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_iv": { + "name": "bot_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_tag": { + "name": "bot_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_id": { + "name": "api_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_ciphertext": { + "name": "api_key_secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_iv": { + "name": "api_key_secret_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_tag": { + "name": "api_key_secret_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_reason": { + "name": "revoked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "slack_workspaces_team_id_idx": { + "name": "slack_workspaces_team_id_idx", + "columns": [ + { + "expression": "team_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_workspaces_org_idx": { + "name": "slack_workspaces_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_workspaces_active_org_idx": { + "name": "slack_workspaces_active_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"slack_workspaces\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_commits": { + "name": "vcs_commits", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_name": { + "name": "author_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_email": { + "name": "author_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_avatar_url": { + "name": "author_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authored_at": { + "name": "authored_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "committed_at": { + "name": "committed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_commits_repo_sha_idx": { + "name": "vcs_commits_repo_sha_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_commits_org_sha_idx": { + "name": "vcs_commits_org_sha_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_installations": { + "name": "vcs_installations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_installation_id": { + "name": "external_installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_login": { + "name": "account_login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_account_id": { + "name": "external_account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_avatar_url": { + "name": "account_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repository_selection": { + "name": "repository_selection", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_installations_provider_external_idx": { + "name": "vcs_installations_provider_external_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_installations_org_idx": { + "name": "vcs_installations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repositories": { + "name": "vcs_repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "tracked_branch": { + "name": "tracked_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_archived": { + "name": "is_archived", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repositories_org_repo_idx": { + "name": "vcs_repositories_org_repo_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_repo_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_org_idx": { + "name": "vcs_repositories_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_installation_idx": { + "name": "vcs_repositories_installation_idx", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repository_branches": { + "name": "vcs_repository_branches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repository_branches_repo_name_idx": { + "name": "vcs_repository_branches_repo_name_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repository_branches_org_idx": { + "name": "vcs_repository_branches_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 5a96dd045..fd8ed4cf8 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -372,6 +372,13 @@ "when": 1788259897779, "tag": "0053_digest_opt_out", "breakpoints": true + }, + { + "idx": 53, + "version": "7", + "when": 1788285534887, + "tag": "0054_alert_incident_open_uniqueness", + "breakpoints": true } ] } diff --git a/packages/db/scripts/backfill-dashboard-datasource-v3.test.ts b/packages/db/scripts/backfill-dashboard-datasource-v3.test.ts new file mode 100644 index 000000000..f60973057 --- /dev/null +++ b/packages/db/scripts/backfill-dashboard-datasource-v3.test.ts @@ -0,0 +1,234 @@ +// BOUNDARY: Test doubles preserve opaque values so the consuming boundary can be exercised. +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { Array as Arr, Option } from "effect" +import type postgres from "postgres" +import { describe, expect, it } from "vitest" +import { + backfillDashboards, + backfillVersionSnapshots, + openJournal, + restore, + type Args, + type JournalLine, + type RecoveryJournal, +} from "./backfill-dashboard-datasource-v3" + +// A minimal v2 document (route data source) that `classify` upgrades and +// decodes cleanly — mirrors the fixture in packages/widgets' upgrade tests. +const v2Document = { + id: "3f1b7c62-5a1e-4d0f-9a3b-6c2e8d4f1a90", + schemaVersion: 2, + name: "Board", + timeRange: { type: "relative", value: "1h" }, + widgets: [ + { + id: "widget-1", + visualization: "chart", + dataSource: { endpoint: "service_overview", params: { serviceName: "api" } }, + display: { title: "T" }, + layout: { x: 0, y: 0, w: 6, h: 4 }, + }, + ], + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", +} + +const applyArgs: Args = { + apply: true, + batch: 100, + dump: "unused.jsonl", + quarantine: "unused-quarantine.jsonl", + skipVersions: false, +} + +interface Captured { + readonly text: string + readonly values: unknown[] +} + +/** + * Fake postgres.js client: records every statement (template text with `$` + * where a parameter goes) and an ordered event log, answers via `respond`. + */ +const makeFakeSql = (respond: (query: Captured) => unknown[]) => { + const queries: Captured[] = [] + const events: string[] = [] + const tag = (strings: TemplateStringsArray, ...values: unknown[]) => { + const captured = { text: strings.join("$"), values } + queries.push(captured) + events.push(captured.text.includes("UPDATE") ? "update" : "select") + return Promise.resolve(respond(captured)) + } + const sql = Object.assign(tag, { + json: (value: unknown) => value, + begin: async (callback: (tx: unknown) => Promise) => { + events.push("begin") + await callback(sql) + events.push("commit") + }, + end: async () => {}, + }) + // SAFETY: the script only uses the tagged-template call, `.json`, `.begin` + // and `.end` — exactly the surface this double implements. + return { sql: sql as unknown as postgres.Sql, queries, events } +} + +const noopQuarantine = () => {} + +const recordingJournal = () => { + const lines: JournalLine[] = [] + const journal: RecoveryJournal = { + append: (line) => lines.push(line), + flush: () => {}, + close: () => {}, + count: () => lines.length, + } + return { journal, lines } +} + +describe("openJournal", () => { + it("persists appended lines on flush, before anything else runs", () => { + const dir = mkdtempSync(join(tmpdir(), "backfill-journal-")) + const path = join(dir, "dump.jsonl") + const journal = openJournal(path) + journal.append({ table: "dashboards", org_id: "o1", id: "d1", version: 3, upgraded_json: {} }) + journal.flush() + // The line is on disk NOW — not at process exit. + const written = readFileSync(path, "utf8").trim().split("\n") + expect(written).toHaveLength(1) + const firstLine = Option.getOrThrow(Arr.head(written)) + expect(JSON.parse(firstLine)).toMatchObject({ table: "dashboards", org_id: "o1", id: "d1" }) + journal.close() + }) +}) + +describe("backfillDashboards", () => { + it("flushes the journal before opening the batch transaction", async () => { + const row = { org_id: "o1", id: "d1", version: 4, payload_json: v2Document } + const { sql, events } = makeFakeSql((query) => + query.text.includes("UPDATE") ? [{ id: "d1" }] : query.text.includes("SELECT") ? [row] : [], + ) + // Shares the fake sql's event array so the interleaving is observable. + const lines: JournalLine[] = [] + const journal: RecoveryJournal = { + append: (line) => { + lines.push(line) + events.push("journal-append") + }, + flush: () => events.push("journal-flush"), + close: () => {}, + count: () => lines.length, + } + + const report = await backfillDashboards(sql, applyArgs, journal, noopQuarantine) + const merged = events + + expect(report.converted).toBe(1) + // The preimage (and the upgraded payload restore will verify against) + // hit the fsynced journal BEFORE the transaction that writes the row. + const flushAt = merged.indexOf("journal-flush") + const beginAt = merged.indexOf("begin") + expect(flushAt).toBeGreaterThan(-1) + expect(beginAt).toBeGreaterThan(flushAt) + expect(lines[0]).toMatchObject({ table: "dashboards", org_id: "o1", id: "d1", version: 4 }) + expect(lines[0]?.upgraded_json).toBeDefined() + }) +}) + +describe("backfillVersionSnapshots", () => { + it("journals snapshot preimages and CASes on the original snapshot", async () => { + const row = { org_id: "o1", id: "v1", snapshot_json: v2Document } + const { sql, queries } = makeFakeSql((query) => + query.text.includes("UPDATE") ? [{ id: "v1" }] : query.text.includes("SELECT") ? [row] : [], + ) + const { journal, lines } = recordingJournal() + + const report = await backfillVersionSnapshots(sql, applyArgs, journal, noopQuarantine) + + expect(report.converted).toBe(1) + expect(lines[0]).toMatchObject({ table: "dashboard_versions", org_id: "o1", id: "v1" }) + const update = queries.find((query) => query.text.includes("UPDATE dashboard_versions")) + // The original snapshot is the compare-and-swap condition: a coalesced + // save that rewrote the row between SELECT and UPDATE must miss. + expect(update?.text).toContain("AND snapshot_json = $::jsonb") + expect(update?.values).toContainEqual(row.snapshot_json) + }) + + it("reports a concurrent rewrite as a CAS miss, not a conversion", async () => { + const row = { org_id: "o1", id: "v1", snapshot_json: v2Document } + const { sql } = makeFakeSql((query) => + query.text.includes("UPDATE") ? [] : query.text.includes("SELECT") ? [row] : [], + ) + const { journal } = recordingJournal() + + const report = await backfillVersionSnapshots(sql, applyArgs, journal, noopQuarantine) + + expect(report.converted).toBe(0) + expect(report.casMissed).toBe(1) + }) +}) + +describe("restore", () => { + const writeDump = (lines: ReadonlyArray): string => { + const dir = mkdtempSync(join(tmpdir(), "backfill-restore-")) + const path = join(dir, "dump.jsonl") + writeFileSync(path, `${lines.map((line) => JSON.stringify(line)).join("\n")}\n`) + return path + } + + it("verifies the row still holds the backfill's payload and advances version", async () => { + const preimage = { schemaVersion: 2, widgets: [] } + const upgraded = { schemaVersion: 3, widgets: [] } + const path = writeDump([ + { + table: "dashboards", + org_id: "o1", + id: "d1", + version: 4, + payload_json: preimage, + upgraded_json: upgraded, + }, + ]) + const { sql, queries } = makeFakeSql(() => [{ id: "d1" }]) + + await restore(sql, path) + + const update = queries[0] + // Version N+1 alone is NOT proof the backfill owns the row — a CAS-missed + // backfill leaves the user's edit at N+1. The current payload must equal + // the exact value the backfill wrote. + expect(update?.text).toContain("AND payload_json = $::jsonb") + expect(update?.values).toContainEqual(upgraded) + expect(update?.values).toContain(5) + // The optimistic-concurrency counter is monotonic: restore bumps it + // forward instead of handing back the pre-backfill value a stale tab + // could still CAS against. + expect(update?.text).toContain("version = version + 1") + expect(update?.text).not.toContain("version = $,") + }) + + it("restores dashboard_versions lines against the written snapshot", async () => { + const preimage = { schemaVersion: 2, widgets: [] } + const upgraded = { schemaVersion: 3, widgets: [] } + const path = writeDump([ + { + table: "dashboard_versions", + org_id: "o1", + id: "v1", + snapshot_json: preimage, + upgraded_json: upgraded, + }, + ]) + const { sql, queries } = makeFakeSql(() => [{ id: "v1" }]) + + await restore(sql, path) + + const update = queries[0] + expect(update?.text).toContain("UPDATE dashboard_versions") + expect(update?.text).toContain("AND snapshot_json = $::jsonb") + expect(update?.values).toContainEqual(upgraded) + expect(update?.values).toContainEqual(preimage) + }) +}) diff --git a/packages/db/scripts/backfill-dashboard-datasource-v3.ts b/packages/db/scripts/backfill-dashboard-datasource-v3.ts index ab877fcdd..18543b981 100644 --- a/packages/db/scripts/backfill-dashboard-datasource-v3.ts +++ b/packages/db/scripts/backfill-dashboard-datasource-v3.ts @@ -24,8 +24,16 @@ * - Every row is DECODED as v3 after transform. A row that fails is left byte * identical and reported; writing a document we could not decode is the one * irreversible mistake available here. - * - The pre-write JSONL dump is flushed BEFORE the batch that it covers, so a - * crash mid-run still leaves every already-written row recoverable. + * - The pre-write JSONL journal is APPENDED AND FSYNCED before the batch it + * covers — for dashboards and for dashboard_versions snapshots — so a crash + * mid-run still leaves every already-written row recoverable. Each line + * carries the preimage AND the exact upgraded payload about to be written: + * restore only touches a row that still holds that exact payload, so a + * CAS-missed backfill row (whose version N+1 belongs to a concurrent user + * edit) is skipped instead of having the user's edit replaced by the stale + * preimage. Restore advances `version` rather than rolling it back — the + * counter is an optimistic-concurrency token and must stay monotonic, or a + * stale tab holding the pre-backfill version could CAS over the restore. * - Writes CAS on `version`. Not bumping it would not avoid disturbing clients * (the change streams over Electric either way) — it would let a stale tab * win the compare-and-swap and silently overwrite the backfill with v2. @@ -45,6 +53,7 @@ * the version-history page, not merely restore. They are not Electric-streamed, * so they carry no open-tab risk. */ +import { closeSync, fsyncSync, openSync, readFileSync, writeSync } from "node:fs" import { Option, Schema } from "effect" import postgres from "postgres" import { @@ -78,7 +87,7 @@ const decodeIssue = (payload: unknown): string => { return exit._tag === "Failure" ? String(exit.cause) : "" } -interface Args { +export interface Args { readonly branch?: string readonly url?: string readonly apply: boolean @@ -134,6 +143,80 @@ interface Row { const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) +/** + * One line of the recovery journal. The preimage plus the exact upgraded + * payload the backfill wrote: `upgraded_json` is what lets restore prove a row + * still holds the backfill's write (and not a user's concurrent edit that won + * the CAS) before reverting it. + */ +export interface JournalLine { + readonly table: "dashboards" | "dashboard_versions" + readonly org_id: string + readonly id: string + /** dashboards only — the pre-backfill CAS token. */ + readonly version?: number + /** dashboards preimage. */ + readonly payload_json?: unknown + /** dashboard_versions preimage. */ + readonly snapshot_json?: unknown + readonly upgraded_json: unknown +} + +/** + * Wire schema for a journal line read back at restore time. `upgraded_json` is + * a REQUIRED key: a line without it predates payload-verified restore, and the + * decode failure is what refuses to apply it. + */ +const JournalLineSchema = Schema.Struct({ + table: Schema.Literals(["dashboards", "dashboard_versions"]), + org_id: Schema.String, + id: Schema.String, + version: Schema.optionalKey(Schema.Number), + payload_json: Schema.optionalKey(Schema.Unknown), + snapshot_json: Schema.optionalKey(Schema.Unknown), + upgraded_json: Schema.Unknown, +}) +const decodeJournalLine = Schema.decodeUnknownOption(JournalLineSchema) + +export interface RecoveryJournal { + readonly append: (line: JournalLine) => void + /** Persist (write + fsync) everything appended so far. */ + readonly flush: () => void + readonly close: () => void + readonly count: () => number +} + +/** + * Append-only JSONL journal. `flush` is called before each batch's writes and + * fsyncs, so an OOM kill, disk error, or crash mid-run cannot lose the + * preimages of rows already committed — the previous version buffered every + * line in memory and wrote once at exit, which is exactly the plan that fails + * when the run does. + */ +export const openJournal = (path: string): RecoveryJournal => { + let fd: number | null = null + let buffered: string[] = [] + let count = 0 + return { + append: (line) => { + buffered.push(JSON.stringify(line)) + count += 1 + }, + flush: () => { + if (buffered.length === 0) return + if (fd === null) fd = openSync(path, "a") + writeSync(fd, `${buffered.join("\n")}\n`) + fsyncSync(fd) + buffered = [] + }, + close: () => { + if (fd !== null) closeSync(fd) + fd = null + }, + count: () => count, + } +} + /** * Classifies a row without writing anything. * @@ -160,10 +243,10 @@ const classify = (payload: unknown) => { return { kind: "converted" as const, upgraded } } -const backfillDashboards = async ( +export const backfillDashboards = async ( sql: postgres.Sql, args: Args, - dump: (line: unknown) => void, + journal: RecoveryJournal, quarantine: (line: unknown) => void, ): Promise => { const report = emptyReport() @@ -198,9 +281,12 @@ const backfillDashboards = async ( } if (args.apply && writes.length > 0) { - // Dump BEFORE the write, so a crash between the two still leaves every - // already-written row recoverable from the file. - for (const { row } of writes) dump(row) + // Journal + fsync BEFORE the write, so a crash between the two still + // leaves every already-written row recoverable from the file. + for (const { row, upgraded } of writes) { + journal.append({ table: "dashboards", ...row, upgraded_json: upgraded }) + } + journal.flush() await sql.begin(async (tx) => { for (const { row, upgraded } of writes) { @@ -226,9 +312,10 @@ const backfillDashboards = async ( return report } -const backfillVersionSnapshots = async ( +export const backfillVersionSnapshots = async ( sql: postgres.Sql, args: Args, + journal: RecoveryJournal, quarantine: (line: unknown) => void, ): Promise => { const report = emptyReport() @@ -245,6 +332,7 @@ const backfillVersionSnapshots = async ( ORDER BY org_id, id LIMIT ${args.batch}` if (rows.length === 0) break + const writes: Array<{ row: (typeof rows)[number]; upgraded: unknown }> = [] for (const row of rows) { report.scanned += 1 const outcome = classify(row.snapshot_json) @@ -261,14 +349,33 @@ const backfillVersionSnapshots = async ( quarantine({ ...row, reason: `snapshot_${outcome.kind}` }) continue } - report.converted += 1 - if (args.apply) { - // No CAS column and no user-visible timestamp here; `created_at` is left - // alone so history keeps its ordering. - await sql` - UPDATE dashboard_versions SET snapshot_json = ${sql.json(outcome.upgraded as never)} - WHERE org_id = ${row.org_id} AND id = ${row.id}` + writes.push({ row, upgraded: outcome.upgraded }) + } + + if (args.apply && writes.length > 0) { + // Snapshot preimages go in the same fsynced journal as dashboards — + // these mutations previously had no rollback path at all. + for (const { row, upgraded } of writes) { + journal.append({ table: "dashboard_versions", ...row, upgraded_json: upgraded }) + } + journal.flush() + for (const { row, upgraded } of writes) { + // No version column here, so the preimage itself is the CAS: the + // persistence service coalesces a fresh save by rewriting this row + // in place, and an unconditioned UPDATE racing it would bury the + // newer user snapshot under an upgrade of the older one. A miss is + // reported, not converted; the re-run classifies the new snapshot + // on its own. `created_at` is left alone so history keeps ordering. + const result = await sql` + UPDATE dashboard_versions SET snapshot_json = ${sql.json(upgraded as never)} + WHERE org_id = ${row.org_id} AND id = ${row.id} + AND snapshot_json = ${sql.json(row.snapshot_json as never)}::jsonb + RETURNING id` + if (result.length === 0) report.casMissed += 1 + else report.converted += 1 } + } else { + report.converted += writes.length } const last = rows[rows.length - 1]! @@ -281,21 +388,42 @@ const backfillVersionSnapshots = async ( } /** - * Restores `payload_json` verbatim from a dump, guarded on `version`. + * Restores preimages from a journal, guarded on the row still holding the + * exact payload the backfill wrote (`upgraded_json`) — `version = N + 1` alone + * is not proof of ownership: when the backfill's CAS missed, N + 1 belongs to + * the user's concurrent edit, and matching on it alone replaced that edit with + * the stale preimage. * - * A row someone has edited since the backfill fails the guard and is reported - * rather than reverted — reverting a user's later edit to undo our own write is - * strictly worse than leaving it. + * A row anyone has edited since the backfill fails the payload guard and is + * reported rather than reverted. Dashboards restores bump `version` forward — + * it is an optimistic-concurrency token, and handing back the pre-backfill + * value would let a stale tab's CAS overwrite the restored state. */ -const restore = async (sql: postgres.Sql, path: string): Promise => { - const text = await Bun.file(path).text() +export const restore = async (sql: postgres.Sql, path: string): Promise => { + const text = readFileSync(path, "utf8") let restored = 0 let skipped = 0 for (const line of text.split("\n").filter((l) => l.trim().length > 0)) { - const row = JSON.parse(line) as Row + const row = Option.getOrElse(decodeJournalLine(JSON.parse(line)), () => + fail( + "Journal line lacks upgraded_json (or is malformed) — this dump predates payload-verified restore and cannot be applied safely.", + ), + ) + if (row.table === "dashboard_versions") { + const result = await sql` + UPDATE dashboard_versions SET snapshot_json = ${sql.json(row.snapshot_json as never)} + WHERE org_id = ${row.org_id} AND id = ${row.id} + AND snapshot_json = ${sql.json(row.upgraded_json as never)}::jsonb + RETURNING id` + if (result.length === 0) skipped += 1 + else restored += 1 + continue + } const result = await sql` - UPDATE dashboards SET payload_json = ${sql.json(row.payload_json as never)}, version = ${row.version} - WHERE org_id = ${row.org_id} AND id = ${row.id} AND version = ${row.version + 1} + UPDATE dashboards SET payload_json = ${sql.json(row.payload_json as never)}, version = version + 1 + WHERE org_id = ${row.org_id} AND id = ${row.id} + AND version = ${(row.version ?? 0) + 1} + AND payload_json = ${sql.json(row.upgraded_json as never)}::jsonb RETURNING id` if (result.length === 0) skipped += 1 else restored += 1 @@ -315,7 +443,7 @@ const printReport = (label: string, report: Report, apply: boolean): void => { const run = async (connectionUrl: string, args: Args): Promise => { const sql = postgres(connectionUrl, { max: 1, prepare: false, onnotice: () => {} }) - const dumpLines: string[] = [] + const journal = openJournal(args.dump) const quarantineLines: string[] = [] try { @@ -324,25 +452,21 @@ const run = async (connectionUrl: string, args: Args): Promise => { return } - const dashboards = await backfillDashboards( - sql, - args, - (line) => dumpLines.push(JSON.stringify(line)), - (line) => quarantineLines.push(JSON.stringify(line)), + const dashboards = await backfillDashboards(sql, args, journal, (line) => + quarantineLines.push(JSON.stringify(line)), ) printReport("dashboards", dashboards, args.apply) let snapshots: Report | null = null if (!args.skipVersions) { - snapshots = await backfillVersionSnapshots(sql, args, (line) => + snapshots = await backfillVersionSnapshots(sql, args, journal, (line) => quarantineLines.push(JSON.stringify(line)), ) printReport("dashboard_versions", snapshots, args.apply) } - if (args.apply && dumpLines.length > 0) { - await Bun.write(args.dump, `${dumpLines.join("\n")}\n`) - console.log(`\n✓ Pre-write dump: ${args.dump} (${dumpLines.length} rows)`) + if (args.apply && journal.count() > 0) { + console.log(`\n✓ Pre-write journal: ${args.dump} (${journal.count()} rows)`) console.log(" Contains customer SQL and query definitions — treat as production data.") } if (quarantineLines.length > 0) { @@ -357,17 +481,21 @@ const run = async (connectionUrl: string, args: Args): Promise => { fail(`${dashboards.quarantined} live dashboard(s) quarantined — inspect ${args.quarantine}`) } } finally { + journal.close() await sql.end({ timeout: 5 }) } } -const args = parseArgs(process.argv.slice(2)) -if (args.url === undefined && args.branch === undefined) { - fail("Pass --branch (PlanetScale) or --url (local rehearsal).") -} - -if (args.url !== undefined) { - await run(args.url, args) -} else { - await withBranchConnection(args.branch!, (url) => run(url, args)) +// CLI entry (skipped when the exports above are imported by tests). +if (import.meta.main) { + const args = parseArgs(process.argv.slice(2)) + const url = Option.fromUndefinedOr(args.url) + const branch = Option.fromUndefinedOr(args.branch) + if (Option.isSome(url)) { + await run(url.value, args) + } else if (Option.isSome(branch)) { + await withBranchConnection(branch.value, (connectionUrl) => run(connectionUrl, args)) + } else { + fail("Pass --branch (PlanetScale) or --url (local rehearsal).") + } } diff --git a/packages/db/scripts/ensure-privileges.test.ts b/packages/db/scripts/ensure-privileges.test.ts new file mode 100644 index 000000000..337b3d1e6 --- /dev/null +++ b/packages/db/scripts/ensure-privileges.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest" +import { defaultPrivilegeStatements, sweepStatements } from "./ensure-privileges" + +describe("defaultPrivilegeStatements", () => { + it("keys table and sequence defaults to the given creating role", () => { + const statements = defaultPrivilegeStatements("postgres") + expect(statements).toHaveLength(2) + for (const statement of statements) { + expect(statement).toContain('ALTER DEFAULT PRIVILEGES FOR ROLE "postgres" IN SCHEMA public') + expect(statement).toContain("TO PUBLIC") + } + }) + + // The standalone path keys defaults per candidate creating role (login role + // AND postgres) — the helper must therefore be callable per role, with the + // dotted PlanetScale login role quoted as one identifier. + it("quotes a dotted PlanetScale login role", () => { + const statements = defaultPrivilegeStatements("pscale_api_abc.def") + expect(statements[0]).toContain('FOR ROLE "pscale_api_abc.def"') + }) +}) + +describe("sweepStatements", () => { + it("backfills existing tables and sequences to PUBLIC", () => { + expect(sweepStatements.some((s) => s.includes("ON ALL TABLES IN SCHEMA public"))).toBe(true) + expect(sweepStatements.some((s) => s.includes("ON ALL SEQUENCES IN SCHEMA public"))).toBe(true) + }) +}) diff --git a/packages/db/scripts/ensure-privileges.ts b/packages/db/scripts/ensure-privileges.ts index 211e531a3..d7f272b73 100644 --- a/packages/db/scripts/ensure-privileges.ts +++ b/packages/db/scripts/ensure-privileges.ts @@ -33,12 +33,14 @@ * migration that DROPped its grants. The sweep below is kept only to heal * objects that predate this — it is idempotent and cheap. * - * Default privileges are keyed to the CREATING role, so this pins the session to - * `postgres` first (see `pinSessionRoleToPostgres` in planetscale-connection.ts, - * which does the same for the brokered prod connection). Where that is not - * permitted we fall back to keying them to the session's own role, which is - * correct for stg / PR previews — there the migrate role and the runtime role - * are the same identity. + * Default privileges are keyed to the CREATING role, and the standalone path + * cannot know which identity a later `drizzle-kit migrate` process will create + * objects as — its own `SET ROLE postgres` is session-scoped, and only the + * brokered prod connection persists the pin (`pinSessionRoleToPostgres` in + * planetscale-connection.ts). So defaults are keyed to BOTH candidates: the + * login role (while the session still is it), then `postgres` where membership + * allows the switch. Whichever one migrate's connections end up creating as, + * its defaults fire. */ import * as Predicate from "effect/Predicate" import postgres from "postgres" @@ -63,19 +65,30 @@ const quoteIdent = (role: string): string => { * Statements are ordered: schema usage, then the default privileges that make * FUTURE objects correct, then the backfill sweep for existing ones. * + * Defaults are keyed to EVERY role migrations might create objects as, not + * just one: this script's `SET ROLE postgres` lasts only for its own session, + * and `drizzle-kit migrate` runs later as a separate process whose connections + * authenticate as the login role. Unless that login carries a persisted + * `role=postgres` (the brokered prod path's `ALTER ROLE … SET role`, see + * planetscale-connection.ts — the standalone stg path has no such guarantee), + * its objects are created by the login role and postgres-keyed defaults never + * fire — recreating exactly the owner-only-table outage this script prevents. + * * PUBLIC is a keyword, not an identifier — it must never be quoted. */ -const statements = (owner: string): readonly string[] => { +export const defaultPrivilegeStatements = (owner: string): readonly string[] => { const ident = quoteIdent(owner) return [ - "GRANT USAGE ON SCHEMA public TO PUBLIC", `ALTER DEFAULT PRIVILEGES FOR ROLE ${ident} IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO PUBLIC`, `ALTER DEFAULT PRIVILEGES FOR ROLE ${ident} IN SCHEMA public GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO PUBLIC`, - "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO PUBLIC", - "GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO PUBLIC", ] } +export const sweepStatements: readonly string[] = [ + "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO PUBLIC", + "GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO PUBLIC", +] + /** Ask the server — a PlanetScale URL username carries a routing suffix that is * stripped before the server sees it, so it is not a usable role name. */ const currentUser = async (sql: postgres.Sql): Promise => { @@ -94,20 +107,35 @@ const currentUser = async (sql: postgres.Sql): Promise => { export const ensureRuntimePrivileges = async (connectionUrl: string): Promise => { const sql = postgres(connectionUrl, { max: 1, fetch_types: false }) try { - // Own the objects as `postgres` where membership allows it, so the default - // privileges are keyed to the role that will actually create prod's tables. + const runStatements = async (batch: readonly string[]): Promise => { + for (const statement of batch) { + console.log(` → ${statement}`) + await sql.unsafe(statement) + } + } + + await runStatements(["GRANT USAGE ON SCHEMA public TO PUBLIC"]) + // Key defaults to the LOGIN role first, while the session still is it: + // `ALTER DEFAULT PRIVILEGES FOR ROLE x` needs membership in x, and + // `postgres` is not a member of its own members. + const loginRole = await currentUser(sql) + console.log(`→ Ensuring PUBLIC privileges for objects created by "${loginRole}"`) + await runStatements(defaultPrivilegeStatements(loginRole)) + // Then as `postgres` where membership allows it, so the defaults are also + // keyed to the role that creates prod's tables (the brokered path pins + // migrations to run as postgres — see planetscale-connection.ts). try { await sql.unsafe("SET ROLE postgres") } catch { - console.log("→ SET ROLE postgres not permitted — keying defaults to the session role") + console.log("→ SET ROLE postgres not permitted — defaults keyed to the session role only") } const owner = await currentUser(sql) - console.log(`→ Ensuring PUBLIC privileges (objects created by "${owner}")\n`) - for (const statement of statements(owner)) { - console.log(` → ${statement}`) - await sql.unsafe(statement) + if (owner !== loginRole) { + console.log(`→ Ensuring PUBLIC privileges for objects created by "${owner}"`) + await runStatements(defaultPrivilegeStatements(owner)) } - console.log(`\n✓ Privileges ensured — future tables owned by "${owner}" are granted to PUBLIC`) + await runStatements(sweepStatements) + console.log(`\n✓ Privileges ensured — future tables are granted to PUBLIC at creation`) } finally { await sql.end() } diff --git a/packages/db/scripts/reset-preview-branch.test.ts b/packages/db/scripts/reset-preview-branch.test.ts new file mode 100644 index 000000000..b77214f5d --- /dev/null +++ b/packages/db/scripts/reset-preview-branch.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest" +import { isPreviewBranchName, resetGuardError } from "./reset-preview-branch" + +describe("resetGuardError", () => { + it("refuses when nothing asserts the target is a preview branch", () => { + expect(resetGuardError({})).toContain("Refusing to reset") + }) + + // THE REGRESSION: the old guard was `if (!process.env.CI && ...)`, so ANY + // nonempty CI value — including the string "false" — authorized dropping + // every object in whatever DATABASE_URL happened to point at. + it("a generic CI flag is not authorization", () => { + expect(resetGuardError({ RESET_EXPECTED_BRANCH: undefined })).not.toBeNull() + // Simulate the old bypass inputs: nothing but CI-ish env noise. + expect(resetGuardError({ RESET_EXPECTED_BRANCH: "" })).not.toBeNull() + }) + + it("proceeds when the caller names a pr-* branch", () => { + expect(resetGuardError({ RESET_EXPECTED_BRANCH: "pr-1234" })).toBeNull() + }) + + it("refuses a non-preview branch name, even in CI", () => { + expect(resetGuardError({ RESET_EXPECTED_BRANCH: "main" })).not.toBeNull() + expect(resetGuardError({ RESET_EXPECTED_BRANCH: "stg" })).not.toBeNull() + expect(resetGuardError({ RESET_EXPECTED_BRANCH: "pr-" })).not.toBeNull() + expect(resetGuardError({ RESET_EXPECTED_BRANCH: "xpr-12" })).not.toBeNull() + }) + + it("still allows an explicit manual confirmation", () => { + expect(resetGuardError({ RESET_PREVIEW_CONFIRM: "1" })).toBeNull() + expect(resetGuardError({ RESET_PREVIEW_CONFIRM: "true" })).not.toBeNull() + }) +}) + +describe("isPreviewBranchName", () => { + it("accepts only pr-", () => { + expect(isPreviewBranchName("pr-7")).toBe(true) + expect(isPreviewBranchName("pr-007")).toBe(true) + expect(isPreviewBranchName("main")).toBe(false) + expect(isPreviewBranchName(undefined)).toBe(false) + }) +}) diff --git a/packages/db/scripts/reset-preview-branch.ts b/packages/db/scripts/reset-preview-branch.ts index 49a9abf59..1bb2d46c8 100644 --- a/packages/db/scripts/reset-preview-branch.ts +++ b/packages/db/scripts/reset-preview-branch.ts @@ -46,7 +46,9 @@ * Exits non-zero if the reset cannot be completed; the caller falls back to * the slow delete + recreate path, so failure here costs time, not correctness. * - * Usage: DATABASE_URL="$MAPLE_PG_URL" bun run --cwd packages/db db:reset-preview + * Usage: RESET_EXPECTED_BRANCH=pr- DATABASE_URL="" \ + * bun run --cwd packages/db db:reset-preview + * (or RESET_PREVIEW_CONFIRM=1 for a manual run — see {@link resetGuardError}) */ import postgres from "postgres" @@ -70,19 +72,44 @@ const quoteIdent = (name: string): string => { return `"${name}"` } +/** Only `pr-` PlanetScale branches are ever a legitimate reset target. */ +export const isPreviewBranchName = (name: string | undefined): boolean => + name !== undefined && /^pr-\d+$/.test(name) + +/** + * Tripwire: this script empties whatever DATABASE_URL points at, and the + * connected role inherits `postgres`, so nothing downstream would stop it from + * gutting prod/stg. The caller must therefore assert WHAT it is resetting — + * `RESET_EXPECTED_BRANCH=pr-` (planetscale-pr-branch.ts sets it from the + * branch it minted the credential for) — or a human must set + * `RESET_PREVIEW_CONFIRM=1`. The old guard keyed off `process.env.CI`, which + * any CI environment sets (`CI=false` included — it is a nonempty string), so + * one copied job or secret mix-up away from wiping a real database. A generic + * environment flag is not authorization for a destructive target. + * + * Returns the refusal message, or `null` to proceed. + */ +export const resetGuardError = (env: { + readonly RESET_EXPECTED_BRANCH?: string | undefined + readonly RESET_PREVIEW_CONFIRM?: string | undefined +}): string | null => { + if (env.RESET_PREVIEW_CONFIRM === "1") return null + if (isPreviewBranchName(env.RESET_EXPECTED_BRANCH?.trim())) return null + return ( + "Refusing to reset: this drops ALL objects in the target database. " + + "Set RESET_EXPECTED_BRANCH=pr- to name the disposable preview branch DATABASE_URL points at " + + "(planetscale-pr-branch.ts does this), or RESET_PREVIEW_CONFIRM=1 for a manual run." + ) +} + const main = async () => { const url = process.env.DATABASE_URL?.trim() if (!url) { fail("DATABASE_URL is not set — pass the PR branch connection string (MAPLE_PG_URL)") } - // Tripwire: this script empties whatever DATABASE_URL points at, and the - // connected role inherits `postgres`, so nothing downstream would stop it - // from gutting prod/stg. It is only ever meant for ephemeral PR-preview - // branches, driven by CI. - if (!process.env.CI && process.env.RESET_PREVIEW_CONFIRM !== "1") { - fail( - "Refusing to reset: this drops ALL objects in the target database. Set RESET_PREVIEW_CONFIRM=1 to confirm DATABASE_URL points at a disposable preview branch (CI runs set CI).", - ) + const guardError = resetGuardError(process.env) + if (guardError !== null) { + fail(guardError) } const sql = postgres(url as string, { max: 1, fetch_types: false }) try { @@ -282,18 +309,21 @@ const main = async () => { } } -try { - await main() -} catch (error) { - // postgres.js errors carry the failing statement — surface it, because the - // server's own error report doesn't. This is how the `malformed array - // literal: ""` failure was pinned to the empty-array bind parameter - // (`!= ALL(${[]})` under fetch_types:false — run 30086768041's - // `parameters: ["pg\\_%", ""]`); keep it so the next mystery identifies - // itself too. - const query = (error as { query?: unknown }).query - const parameters = (error as { parameters?: unknown }).parameters - if (query !== undefined) console.error(`✗ failing query: ${String(query).slice(0, 500)}`) - if (parameters !== undefined) console.error(`✗ query parameters: ${JSON.stringify(parameters)}`) - throw error +// CLI entry (skipped when the guard exports above are imported by tests). +if (import.meta.main) { + try { + await main() + } catch (error) { + // postgres.js errors carry the failing statement — surface it, because the + // server's own error report doesn't. This is how the `malformed array + // literal: ""` failure was pinned to the empty-array bind parameter + // (`!= ALL(${[]})` under fetch_types:false — run 30086768041's + // `parameters: ["pg\\_%", ""]`); keep it so the next mystery identifies + // itself too. + const query = (error as { query?: unknown }).query + const parameters = (error as { parameters?: unknown }).parameters + if (query !== undefined) console.error(`✗ failing query: ${String(query).slice(0, 500)}`) + if (parameters !== undefined) console.error(`✗ query parameters: ${JSON.stringify(parameters)}`) + throw error + } } diff --git a/packages/db/src/schema/alerts.ts b/packages/db/src/schema/alerts.ts index 8d6852d99..1eb7ad502 100644 --- a/packages/db/src/schema/alerts.ts +++ b/packages/db/src/schema/alerts.ts @@ -10,6 +10,7 @@ import { timestamp, uniqueIndex, } from "drizzle-orm/pg-core" +import { sql } from "drizzle-orm" import type { AlertDeliveryEventId, AlertDestinationId, @@ -192,6 +193,14 @@ export const alertIncidents = pgTable( index("alert_incidents_org_rule_idx").on(table.orgId, table.ruleId), index("alert_incidents_org_issue_idx").on(table.orgId, table.errorIssueId), uniqueIndex("alert_incidents_incident_key_idx").on(table.incidentKey), + // One open incident per (rule, group): the scheduler's claim serializes + // rule evaluation in the common case, and this makes duplicate opens from + // an expired claim impossible instead of merely unlikely. NULL groupKey + // rows (pre-scheduler history) escape the constraint; the scheduler always + // writes a string key. + uniqueIndex("alert_incidents_open_group_idx") + .on(table.orgId, table.ruleId, table.groupKey) + .where(sql`${table.status} = 'open'`), ], ) diff --git a/packages/db/src/schema/anomalies.ts b/packages/db/src/schema/anomalies.ts index afb08874a..0ccd724aa 100644 --- a/packages/db/src/schema/anomalies.ts +++ b/packages/db/src/schema/anomalies.ts @@ -8,6 +8,7 @@ import { primaryKey, text, timestamp, + uniqueIndex, } from "drizzle-orm/pg-core" import { sql } from "drizzle-orm" import type { AnomalyIncidentId, ErrorIssueId, OrgId, UserId } from "@maple/domain/primitives" @@ -133,6 +134,13 @@ export const anomalyIncidents = pgTable( index("anomaly_incidents_org_triggered_idx").on(table.orgId, table.lastTriggeredAt), index("anomaly_incidents_org_detector_idx").on(table.orgId, table.detectorKey), index("anomaly_incidents_org_issue_idx").on(table.orgId, table.errorIssueId), + // One open incident per detector: the org claim is a bare lastTickAt CAS + // with no renewal, so a tick that outruns ORG_LOCK_TTL_MS can overlap the + // next one — this turns the duplicate open into a no-op/loud conflict + // instead of two incidents, two triages, two pages. + uniqueIndex("anomaly_incidents_open_detector_idx") + .on(table.orgId, table.detectorKey) + .where(sql`${table.status} = 'open'`), ], ) diff --git a/packages/db/src/schema/vcs.ts b/packages/db/src/schema/vcs.ts index 80adf5301..9811e0098 100644 --- a/packages/db/src/schema/vcs.ts +++ b/packages/db/src/schema/vcs.ts @@ -59,6 +59,8 @@ export const vcsRepositories = pgTable( provider: text("provider").$type().notNull(), // Internal id of the owning vcs_installations row (NOT the provider's external // installation id). Provider ids are resolved at the sync/webhook boundary. + // Deliberately no FK (house style): VcsRepository enforces the link — child + // upserts share-lock the parent row and no-op when it is gone. installationId: text("installation_id").$type().notNull(), externalRepoId: text("external_repo_id").notNull(), owner: text("owner").notNull(), @@ -92,12 +94,15 @@ export const vcsRepositories = pgTable( /** * Resolved commits. Each commit belongs to exactly one `vcs_repositories` row - * (`repository_id`) — a commit without a repo is not meaningful, and that link - * is what a repo/installation purge cascades on. There is no branch link: a repo + * (`repository_id`) — a commit without a repo is not meaningful. There is no FK: + * VcsRepository enforces the link (purges delete children in the same + * transaction; child upserts share-lock the parent and no-op when it is gone, + * and the (org_id, sha) reads join the parent). There is no branch link: a repo * stores the commits of its single tracked branch, so "the repo's commits" is the * whole set. The dashboard resolver matches a trace's full 40-char SHA by - * `(org_id, sha)` — provider-agnostic, no join — so `org_id` stays denormalized - * here. The row is self-contained (`html_url` + author fields). + * `(org_id, sha)` — provider-agnostic, and `org_id` stays denormalized here so + * the lookup needs only the orphan-shield join described above. The row is + * self-contained (`html_url` + author fields). */ export const vcsCommits = pgTable( "vcs_commits", diff --git a/packages/domain/src/clickhouse/migrations/0009_one_year_service_history.ts b/packages/domain/src/clickhouse/migrations/0009_one_year_service_history.ts index 7139d4abb..adc29303d 100644 --- a/packages/domain/src/clickhouse/migrations/0009_one_year_service_history.ts +++ b/packages/domain/src/clickhouse/migrations/0009_one_year_service_history.ts @@ -140,9 +140,15 @@ ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(Hour) ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) TTL toDate(Hour) + INTERVAL 365 DAY`, - serviceOverviewHourlyBackfill, - serviceOperationsHourlyBackfill, + // DROP → TRUNCATE → backfill → CREATE MV, the 0015 cutover shape. The + // truncates make a re-apply of a partially failed run converge instead of + // doubling the additive sum aggregates; safe only because both views are + // detached first, so nothing writes the targets during the backfills. "DROP VIEW IF EXISTS service_overview_hourly_mv", + "DROP VIEW IF EXISTS service_operations_hourly_mv", + "TRUNCATE TABLE IF EXISTS service_overview_hourly", + "TRUNCATE TABLE IF EXISTS service_operations_hourly", + serviceOverviewHourlyBackfill, `CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS SELECT OrgId, @@ -163,7 +169,7 @@ SELECT FROM traces WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha`, - "DROP VIEW IF EXISTS service_operations_hourly_mv", + serviceOperationsHourlyBackfill, `CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS SELECT OrgId, diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 22967be77..d1b280636 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -1,3 +1,4 @@ +import { String as Str } from "effect" import { describe, expect, it } from "vitest" import { type BackfillSpec, isBackfill, renderStatementFull } from "../backfill" import { migration_0004_service_namespace_projections } from "./0004_service_namespace_projections" @@ -387,6 +388,72 @@ describe("ClickHouse migrations", () => { expect(serviceOperationsHourlyBackfill.tsColumn).toBe("Minute") }) + it("orders 0009 so a re-apply converges instead of doubling the annual rollups", () => { + // The targets are additive AggregatingMergeTrees retained a year: replaying + // the backfills after a partial failure (chunk N fails, admin re-runs the + // apply) would double every sum until TTL. DROP → TRUNCATE → backfill → + // CREATE MV is the 0015 cutover shape, restated as invariants. + const kinds = migration_0009_one_year_service_history.statements.map((stmt) => + // Str.split returns a NonEmptyArray, so the head index is statically safe. + isBackfill(stmt) ? `backfill:${stmt.target}` : Str.split(stmt, "\n")[0].trim(), + ) + const at = (needle: string) => { + const index = kinds.findIndex((kind) => kind.startsWith(needle)) + expect(index, needle).toBeGreaterThanOrEqual(0) + return index + } + + // Both live writers are detached before either target is truncated. + expect(at("DROP VIEW IF EXISTS service_overview_hourly_mv")).toBeLessThan( + at("TRUNCATE TABLE IF EXISTS service_overview_hourly"), + ) + expect(at("DROP VIEW IF EXISTS service_operations_hourly_mv")).toBeLessThan( + at("TRUNCATE TABLE IF EXISTS service_operations_hourly"), + ) + // Each target is emptied before its backfill, and its view reattaches after. + expect(at("TRUNCATE TABLE IF EXISTS service_overview_hourly")).toBeLessThan( + at("backfill:service_overview_hourly"), + ) + expect(at("backfill:service_overview_hourly")).toBeLessThan( + at("CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv"), + ) + expect(at("TRUNCATE TABLE IF EXISTS service_operations_hourly")).toBeLessThan( + at("backfill:service_operations_hourly"), + ) + expect(at("backfill:service_operations_hourly")).toBeLessThan( + at("CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv"), + ) + }) + + it("keeps every backfill convergent on re-apply: target emptied or rebuilt first", () => { + // A migration is recorded only after every statement succeeds, so a failure + // anywhere replays the WHOLE migration — including backfills that already + // inserted. Every backfill target must therefore be emptied of the rows the + // backfill writes earlier in the same migration: truncated, rebuilt from + // scratch (DROP TABLE IF EXISTS + rename swap), or scoped-deleted + // for a dual-fed target (0021). An additive INSERT…SELECT into a surviving + // table doubles on the rerun and nothing downstream can detect it. + // identity_links is exempt because a replay is a merge no-op: its only + // aggregate is SimpleAggregateFunction(min) keyed by the full sorting key, + // so re-inserted rows collapse to the values already stored. + const mergeIdempotentTargets = new Set(["identity_links"]) + for (const migration of migrations) { + migration.statements.forEach((stmt, index) => { + if (!isBackfill(stmt) || mergeIdempotentTargets.has(stmt.target)) return + const before = migration.statements + .slice(0, index) + .filter((s): s is string => typeof s === "string") + const emptied = before.some( + (s) => + s.startsWith(`TRUNCATE TABLE IF EXISTS ${stmt.target}`) || + s.startsWith(`DROP TABLE IF EXISTS ${stmt.target}`) || + s.startsWith(`DELETE FROM ${stmt.target} `), + ) + expect(emptied, `m${migration.version} backfill:${stmt.target}`).toBe(true) + }) + } + }) + it("adds the service-operation rollup and exposes a coordinated chunkable backfill", () => { const statements = migration_0008_service_operations_minutely.statements const sql = statements.map((statement) => renderStatementFull(statement, "default")).join("\n\n") diff --git a/packages/domain/src/http/org-clickhouse-settings.ts b/packages/domain/src/http/org-clickhouse-settings.ts index c0adce14e..fa3b1c55c 100644 --- a/packages/domain/src/http/org-clickhouse-settings.ts +++ b/packages/domain/src/http/org-clickhouse-settings.ts @@ -174,6 +174,18 @@ export class OrgClickHouseApplySchemaStatus extends Schema.Class 0, sum(Sum) / sum(Count), 0) AS avgValue, - min(Min) AS minValue, - max(Max) AS maxValue, + ifNull(min(Min), 0) AS minValue, + ifNull(max(Max), 0) AS maxValue, sum(Sum) AS sumValue, sum(Count) AS dataPointCount FROM metrics_histogram @@ -9680,15 +9680,15 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- spec:metrics-timeseries-grouped-by-resource:baseline [b9a2d4d6] +-- spec:metrics-timeseries-grouped-by-resource:baseline [c0ea8282] SELECT toStartOfInterval(TimeUnix, INTERVAL 300 SECOND) AS bucket, ServiceName AS serviceName, ResourceAttributes['host.name'] AS attributeValue, ResourceAttributes['host.name'] AS groupName, if(sum(Count) > 0, sum(Sum) / sum(Count), 0) AS avgValue, - min(Min) AS minValue, - max(Max) AS maxValue, + ifNull(min(Min), 0) AS minValue, + ifNull(max(Max), 0) AS maxValue, sum(Sum) AS sumValue, sum(Count) AS dataPointCount FROM metrics_histogram @@ -9732,15 +9732,15 @@ SELECT ORDER BY bucket ASC FORMAT JSON --- spec:metrics-timeseries:baseline [e6156cc2] +-- spec:metrics-timeseries:baseline [8a2c3b82] SELECT toStartOfInterval(TimeUnix, INTERVAL 3600 SECOND) AS bucket, ServiceName AS serviceName, '' AS attributeValue, ServiceName AS groupName, if(sum(Count) > 0, sum(Sum) / sum(Count), 0) AS avgValue, - min(Min) AS minValue, - max(Max) AS maxValue, + ifNull(min(Min), 0) AS minValue, + ifNull(max(Max), 0) AS maxValue, sum(Sum) AS sumValue, sum(Count) AS dataPointCount FROM metrics_histogram diff --git a/packages/query-engine/src/ch/queries/metrics.test.ts b/packages/query-engine/src/ch/queries/metrics.test.ts index e699e2d6c..fb1595ae0 100644 --- a/packages/query-engine/src/ch/queries/metrics.test.ts +++ b/packages/query-engine/src/ch/queries/metrics.test.ts @@ -62,8 +62,10 @@ describe("metricsTimeseriesQuery", () => { const { sql } = compileUnsafe(q, baseParams) expect(sql).toContain("FROM metrics_histogram") expect(sql).toContain("sum(Sum) / sum(Count)") - expect(sql).toContain("min(Min) AS minValue") - expect(sql).toContain("max(Max) AS maxValue") + // Nullable extrema fall back to 0 so an all-NULL bucket still decodes + // through the non-null Float64 row contract. + expect(sql).toContain("ifNull(min(Min), 0) AS minValue") + expect(sql).toContain("ifNull(max(Max), 0) AS maxValue") expect(sql).toContain("sum(Sum) AS sumValue") expect(sql).toContain("sum(Count) AS dataPointCount") }) diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts index 142cc2a68..62973fa80 100644 --- a/packages/query-engine/src/ch/queries/query-helpers.ts +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -582,8 +582,11 @@ export function metricsSelectExprs($: ColumnAccessor, const $h = $ as unknown as ColumnAccessor return { avgValue: CH.if_(CH.sum($h.Count).gt(0), CH.sum($h.Sum).div(CH.sum($h.Count)), CH.lit(0)), - minValue: CH.min_($h.Min), - maxValue: CH.max_($h.Max), + // Min/Max are Nullable (OTel histograms may omit extrema), and min/max + // over an all-NULL bucket return NULL — fall back to 0 like avgValue so + // the declared non-null Float64 row contract holds. + minValue: CH.ifNull(CH.min_($h.Min), CH.lit(0)), + maxValue: CH.ifNull(CH.max_($h.Max), CH.lit(0)), sumValue: CH.sum($h.Sum), dataPointCount: CH.sum($h.Count), } diff --git a/packages/query-engine/src/ch/queries/service-map-rollup.test.ts b/packages/query-engine/src/ch/queries/service-map-rollup.test.ts index 1f0a15038..957b742bb 100644 --- a/packages/query-engine/src/ch/queries/service-map-rollup.test.ts +++ b/packages/query-engine/src/ch/queries/service-map-rollup.test.ts @@ -3,6 +3,7 @@ import { Effect } from "effect" import { serviceMapEdgesExistingHoursSQL, serviceMapEdgesRollupSQL, + serviceMapResolutionsExistingHoursSQL, serviceMapResolutionsRollupSQL, } from "./service-map-rollup" @@ -97,3 +98,33 @@ describe("service-map rollup compiled row schemas", () => { }), ) }) + +describe("service-map rollup routing", () => { + const windowParams = { + orgId: "org_1", + startTime: "2024-01-01 00:00:00", + endTime: "2024-01-02 00:00:00", + } + + it.effect("pins both seal probes to the ingest backend the rollup writes", () => + Effect.gen(function* () { + // The rollup ingests into Tinybird unconditionally. A probe resolved as + // an ordinary read hits a BYO org's own (never-written) ClickHouse table, + // judges every hour missing, and re-ingests the same additive edge rows + // into Tinybird every tick — permanent double counting. + const edges = yield* serviceMapEdgesExistingHoursSQL(windowParams) + const resolutions = yield* serviceMapResolutionsExistingHoursSQL(windowParams) + expect(edges.route).toBe("ingest") + expect(resolutions.route).toBe("ingest") + }), + ) + + it.effect("leaves the compute rollups on the org backend where raw spans live", () => + Effect.gen(function* () { + const edges = yield* serviceMapEdgesRollupSQL(hourParams) + const resolutions = yield* serviceMapResolutionsRollupSQL(hourParams) + expect(edges.route).toBeUndefined() + expect(resolutions.route).toBeUndefined() + }), + ) +}) diff --git a/packages/query-engine/src/ch/queries/service-map-rollup.ts b/packages/query-engine/src/ch/queries/service-map-rollup.ts index 17e247f5c..fc3bb25ee 100644 --- a/packages/query-engine/src/ch/queries/service-map-rollup.ts +++ b/packages/query-engine/src/ch/queries/service-map-rollup.ts @@ -95,6 +95,11 @@ export function serviceMapEdgesExistingHoursSQL(params: { ]) .groupBy("hourTs") .format("JSON") + // The seal probe must read the backend the rollup WRITES (`ingest` is + // Tinybird-pinned). Resolved as a read for a BYO-ClickHouse org, it saw + // that org's never-written table, judged every hour missing, and re-rolled + // + re-ingested the same additive rows into Tinybird on every tick. + .route("ingest") return compile(query, { orgId: params.orgId, @@ -128,6 +133,10 @@ export function serviceMapResolutionsExistingHoursSQL(params: { ]) .groupBy("hourTs") .format("JSON") + // Same backend-consistency rule as the edges probe: resolutions are + // written via `ingest`, so "which hours already resolved" must ask the + // ingest backend, not a BYO read override. + .route("ingest") return compile(query, { orgId: params.orgId, diff --git a/packages/widgets/src/dashboard/migrations/migrations.test.ts b/packages/widgets/src/dashboard/migrations/migrations.test.ts index 2427ddaef..e3b231780 100644 --- a/packages/widgets/src/dashboard/migrations/migrations.test.ts +++ b/packages/widgets/src/dashboard/migrations/migrations.test.ts @@ -335,3 +335,80 @@ describe("stampCurrentVersion", () => { expect(encoded.schemaVersion).toBe(CURRENT_DASHBOARD_SCHEMA_VERSION) }) }) + +describe("the v1 -> v2 step recurses into display.sparkline.dataSource", () => { + // The sparkline embeds a FULL v1 data source, open transform included. Left + // unclosed, one legacy value there kept the whole document undecodable under + // v2/v3 — and the writable path then refused the entire dashboard. + const sparklineDocument = { + ...legacyDocument, + widgets: [ + { + ...legacyDocument.widgets[0], + display: { + title: "Requests", + sparkline: { + enabled: true, + dataSource: { + endpoint: "custom_query_builder_timeseries", + params: { + queries: [{ id: "a", name: "A", dataSource: "traces", aggregation: "count" }], + }, + transform: { + reduceToValue: { field: "value", aggregate: "median" }, + sortBy: { field: "value", direction: "descending" }, + }, + }, + }, + }, + }, + ], + } + + it("closes the sparkline's open transform values", () => { + const display = firstWidget(migrateToLatest(sparklineDocument)).display + expect(display).toMatchObject({ + sparkline: { + dataSource: { + transform: { + reduceToValue: { field: "value", aggregate: "first" }, + sortBy: { field: "value", direction: "asc" }, + }, + }, + }, + }) + }) + + it("is idempotent on the sparkline too", () => { + for (const migration of DASHBOARD_MIGRATIONS) { + const once = migration.migrate(sparklineDocument) + expect(migration.migrate(once)).toEqual(once) + } + }) + + it("makes the full document decode through the stored upgrade", () => { + expect(parse(sparklineDocument)._tag).toBe("Decoded") + }) +}) + +describe("upgradeStoredDocument with a document from a newer build", () => { + // `migrateToLatest` refuses to migrate a future document, but the combined + // upgrader used to run the v3 rewrite on it anyway and restamp it 3 — + // erasing the version marker and re-encoding shapes it cannot know. + const fromTheFuture = { + ...legacyDocument, + schemaVersion: 99, + widgets: [ + { + ...legacyDocument.widgets[0], + // A hypothetical future data source with no string `kind`: the v3 + // rewrite would have destructively re-read it as a legacy source. + dataSource: { kind: 7, spec: { future: true } }, + }, + ], + } + + it("returns it untouched instead of downstamping and rewriting it", () => { + expect(upgradeStoredDocument(fromTheFuture)).toEqual(fromTheFuture) + }) +}) diff --git a/packages/widgets/src/dashboard/migrations/v1-to-v2.ts b/packages/widgets/src/dashboard/migrations/v1-to-v2.ts index c16f29453..a01bc4a89 100644 --- a/packages/widgets/src/dashboard/migrations/v1-to-v2.ts +++ b/packages/widgets/src/dashboard/migrations/v1-to-v2.ts @@ -82,6 +82,31 @@ const migrateWidget = (widget: unknown): unknown => { next.dataSource = { ...dataSource, transform: migrateTransform(dataSource.transform) } } + // `display.sparkline.dataSource` embeds a full v1 data source, so its + // transform needs the same closing — left open, one legacy sparkline value + // (`aggregate: "median"`) keeps the whole document undecodable under v2/v3 + // and the writable path then refuses the entire dashboard. + const display = next.display + if (isPlainObject(display)) { + const sparkline = display.sparkline + if ( + isPlainObject(sparkline) && + isPlainObject(sparkline.dataSource) && + sparkline.dataSource.transform !== undefined + ) { + next.display = { + ...display, + sparkline: { + ...sparkline, + dataSource: { + ...sparkline.dataSource, + transform: migrateTransform(sparkline.dataSource.transform), + }, + }, + } + } + } + return next } diff --git a/packages/widgets/src/dashboard/upgrade-to-v3.ts b/packages/widgets/src/dashboard/upgrade-to-v3.ts index cbe29f6ee..0acc3b9ac 100644 --- a/packages/widgets/src/dashboard/upgrade-to-v3.ts +++ b/packages/widgets/src/dashboard/upgrade-to-v3.ts @@ -204,6 +204,14 @@ export const upgradeDocumentToV3 = (document: unknown): unknown => { * `upgradeDocumentToV3` — and by then nothing calls it. */ export const upgradeStoredDocument = (payload: unknown): unknown => { + // A document declaring a NEWER version than this build knows comes back from + // `migrateToLatest` untouched — but the v3 rewrite and the unconditional + // restamp below would then erase its version marker and mangle any data + // source shape v3 cannot know. Pass it through unchanged instead. + if (isRecord(payload)) { + const declared = payload.schemaVersion + if (typeof declared === "number" && declared > CURRENT_DASHBOARD_SCHEMA_VERSION) return payload + } const upgraded = upgradeDocumentToV3(migrateToLatest(payload)) // Restamp here, not in `migrateToLatest`. That function stamps the version it // actually REACHED — which is 2, since the chain stops there — and it is right diff --git a/scripts/planetscale-pr-branch.ts b/scripts/planetscale-pr-branch.ts index 7bf8361cf..b4a582370 100644 --- a/scripts/planetscale-pr-branch.ts +++ b/scripts/planetscale-pr-branch.ts @@ -479,7 +479,7 @@ const createAndAwaitBranch = async (database: string, branchName: string): Promi * packages/db/scripts/reset-preview-branch.ts). Returns false on failure so the * caller can fall back to delete → recreate. */ -const resetBranchInPlace = (connectionUrl: string, replicationUrl?: string): boolean => { +const resetBranchInPlace = (branchName: string, connectionUrl: string, replicationUrl?: string): boolean => { const dbPackageDir = fileURLToPath(new URL("../packages/db", import.meta.url)) console.log(`$ bun run --cwd packages/db db:reset-preview`) const proc = spawnSync("bun", ["run", "--cwd", dbPackageDir, "db:reset-preview"], { @@ -488,6 +488,9 @@ const resetBranchInPlace = (connectionUrl: string, replicationUrl?: string): boo env: { ...process.env, DATABASE_URL: connectionUrl, + // The reset script's guard: name the pr-* branch this credential was + // minted for. A generic CI flag no longer authorizes the wipe. + RESET_EXPECTED_BRANCH: branchName, // The inactive-slot sweep needs the REPLICATION attribute the main // role deliberately lacks. ...(replicationUrl ? { REPLICATION_DATABASE_URL: replicationUrl } : undefined), @@ -542,7 +545,7 @@ const main = async () => { // branch delete revokes both roles and fresh ones are minted after the // recreate. const electric = createCredential(database, branchName, { replication: true, suffix: "-repl" }) - if (resetBranchInPlace(credential.url, electric.url)) { + if (resetBranchInPlace(branchName, credential.url, electric.url)) { maskAndExport({ MAPLE_PG_URL: credential.url, MAPLE_PG_ELECTRIC_URL: electric.url }, [ credential.password, electric.password,