diff --git a/.changeset/member-directory-auth-cutover.md b/.changeset/member-directory-auth-cutover.md new file mode 100644 index 0000000000..dec9f04338 --- /dev/null +++ b/.changeset/member-directory-auth-cutover.md @@ -0,0 +1,11 @@ +--- +"@executor-js/cloud": patch +"@executor-js/api": patch +"@executor-js/host-selfhost": patch +--- + +Cloud now authorizes every protected request against the local membership mirror through the shared `MemberDirectory` seam: the per-request org membership check, the admin gates on the account and admin planes, the org switcher's organization list, and the free-organization limit all read the mirror instead of calling WorkOS. WorkOS is now a write target and an event source only. The seam gains `membershipsOf(accountId)` and `membershipById(organizationId, membershipId)` on both hosts. + +The mirror is trusted only while it is **ready**: the backfill has written every organization and the Events reconciler has drained the stream within the last ten minutes (both recorded on the `workos_sync` row). Until then the membership check falls back to WorkOS, exactly as before, so a member the backfill has not written yet is not locked out and a member revoked while the reconciler was down is not let in. The deploy runs `scripts/ensure-workos-mirror-ready.ts` after the migrations: it runs the backfill if needed, drains the events stream itself if the reconciler has not recently (so the gate never waits on a cron this same deploy ships), and fails the deploy if the mirror is still not ready. An organization the mirror does not hold at all (one that predates the mirror and nobody has signed in to since) is resolved from WorkOS on demand for a caller WorkOS confirms as its member, so CLI and MCP tokens naming such an organization are not refused. Deleting an organization now cancels billing before deleting the WorkOS organization, and a retry after a partial deletion is admitted from the mirror even while the mirror is not ready. + +**Ops step (cloud):** add the `WORKOS_API_KEY` secret to the `production` GitHub environment so the deploy gate can run the backfill. diff --git a/.claude/skills/prod-telemetry/SKILL.md b/.claude/skills/prod-telemetry/SKILL.md index 0128c9ecd0..9af858ca05 100644 --- a/.claude/skills/prod-telemetry/SKILL.md +++ b/.claude/skills/prod-telemetry/SKILL.md @@ -57,6 +57,23 @@ join the same traces via traceparent). `execute`/`execute-action` calls `mcp.execute.code` (the script itself, capped at 10k chars — cloud-only content capture; local/self-host telemetry never records content). +- `auth.authorize_organization` — every membership authorization. + `mirror.ready` (bool: the local membership mirror answered; `false` = + the request fell back to a live WorkOS read) and `mirror.readiness` + (why: `ready`, `backfill pending: …`, `reconciler stale: …`). The + mirror's write spans are `workos_mirror.`; the reconciler run is + `workos_events.sync`. `workos_sync.drained_at` in the prod DB is the + reconciler heartbeat. + +**Recipe — membership-mirror fallback rate (should be ~0 after cutover):** + +```apl +['executor-cloud'] +| where _time > ago(1h) and name == "auth.authorize_organization" +| extend ready = tobool(['attributes.custom']['mirror.ready']) +| extend why = tostring(['attributes.custom']['mirror.readiness']) +| summarize n = count() by ready, why +``` **Recipe — error signatures by class (the daily-digest query):** diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2f0d97c242..12b83997c8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -54,6 +54,19 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + # The build below authorizes every request from the local membership + # mirror. This runs the mirror backfill if it has not completed, drains + # the WorkOS events stream itself if the reconciler has not recently + # (it does not wait on the cron, which this same deploy may be the one + # to ship), and FAILS the deploy if the mirror is still not ready — see + # scripts/ensure-workos-mirror-ready.ts. + - name: Backfill and verify the membership mirror + run: bun run scripts/ensure-workos-mirror-ready.ts + working-directory: apps/cloud + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + WORKOS_API_KEY: ${{ secrets.WORKOS_API_KEY }} + deploy-cloud: name: Deploy cloud runs-on: blacksmith-4vcpu-ubuntu-2404 diff --git a/apps/cloud/package.json b/apps/cloud/package.json index 328d096b52..b7871ba869 100644 --- a/apps/cloud/package.json +++ b/apps/cloud/package.json @@ -36,6 +36,7 @@ "db:backfill-workos-mirror:dev": "op run --env-file=.env.op -- bun run scripts/backfill-workos-mirror.ts", "db:drain-workos-events:prod": "op run --env-file=.env.production -- bun run scripts/drain-workos-events.ts", "db:drain-workos-events:dev": "op run --env-file=.env.op -- bun run scripts/drain-workos-events.ts", + "db:ensure-workos-mirror-ready:prod": "op run --env-file=.env.production -- bun run scripts/ensure-workos-mirror-ready.ts", "routes:gen": "bun scripts/gen-routes.ts", "vendor-wasm": "bun run scripts/vendor-quickjs-wasm.ts" }, diff --git a/apps/cloud/scripts/ensure-workos-mirror-ready.ts b/apps/cloud/scripts/ensure-workos-mirror-ready.ts new file mode 100644 index 0000000000..d10b826d3e --- /dev/null +++ b/apps/cloud/scripts/ensure-workos-mirror-ready.ts @@ -0,0 +1,115 @@ +/* oxlint-disable executor/no-try-catch-or-throw -- boundary: out-of-band deploy gate over a raw postgres connection */ +// --------------------------------------------------------------------------- +// Deploy gate: make the membership mirror READY before the build that +// authorizes from it goes live, and fail the deploy if it cannot be. +// +// bun run db:ensure-workos-mirror-ready:prod # op run --env-file=.env.production +// (deploy.yml runs it after the migrations, before the cloud deploy) +// +// Readiness is the SAME rule the request path applies +// (`src/auth/mirror-readiness-store.ts`): the one-off backfill has written +// every organization (`workos_sync.backfill_completed_at`) AND the events +// reconciler has drained the stream within its lag budget +// (`workos_sync.drained_at`). Until both hold the deployed build reads +// membership from WorkOS instead of the mirror, so an unready mirror never +// locks anyone out or lets a revoked member in — but a deploy that leaves it +// unready would run every request through that fallback, which is the state +// this whole cutover exists to leave behind. So this gate: +// 1. reads the readiness row; +// 2. if the backfill has not completed, RUNS it (scripts/backfill-workos-mirror.ts, +// idempotent) and reads again; +// 3. if the reconciler has not drained recently, DRAINS the stream itself +// (scripts/drain-workos-events.ts: the same replay the Worker's cron +// runs, over this connection) and reads again — never merely waits for +// the cron: this gate runs BEFORE the build that carries the cron may +// have been deployed, and a gate that only waited could not pass until +// the reconciler build had shipped on its own, by hand. A cron that is +// already live is safe beside it (the cursor's compare-and-set gives +// the stream one owner at a time); +// 4. exits 0 only when the mirror is ready, and 1 with the reason otherwise. +// Needs DATABASE_URL and WORKOS_API_KEY (the backfill and the drain read WorkOS). +// --------------------------------------------------------------------------- + +import { spawnSync } from "node:child_process"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { drizzle } from "drizzle-orm/postgres-js"; +import postgres from "postgres"; + +import { + MirrorReadinessState, + describeMirrorReadiness, + readMirrorReadiness, +} from "../src/auth/mirror-readiness-store"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BACKFILL_SCRIPT = resolve(__dirname, "backfill-workos-mirror.ts"); +const DRAIN_SCRIPT = resolve(__dirname, "drain-workos-events.ts"); + +const connectionString = process.env.DATABASE_URL; +if (!connectionString) { + console.error("DATABASE_URL is not set"); + process.exit(1); +} + +const usesLocalDatabase = + connectionString.includes("127.0.0.1") || connectionString.includes("localhost"); + +const sql = postgres(connectionString, { + max: 1, + prepare: false, + ...(usesLocalDatabase ? {} : { ssl: "require" as const }), +}); +const db = drizzle(sql); + +const log = (line: string) => console.log(`[mirror-ready] ${line}`); + +const readiness = () => readMirrorReadiness(db, new Date()); + +// The backfill and drain scripts own their own WorkOS + database wiring; +// running them as subprocesses (with this process's env) keeps that wiring +// in one place. +const runScript = (what: string, script: string) => { + if (!process.env.WORKOS_API_KEY) { + throw new Error(`WORKOS_API_KEY is not set; the mirror ${what} cannot run`); + } + const result = spawnSync("bun", ["run", script], { + stdio: "inherit", + env: process.env, + }); + if (result.status !== 0) { + throw new Error(`the mirror ${what} exited with status ${result.status ?? "unknown"}`); + } +}; + +try { + let state = await readiness(); + log(describeMirrorReadiness(state)); + + if (MirrorReadinessState.$is("BackfillPending")(state)) { + log("backfill not completed; running scripts/backfill-workos-mirror.ts"); + runScript("backfill", BACKFILL_SCRIPT); + state = await readiness(); + log(describeMirrorReadiness(state)); + } + + if (MirrorReadinessState.$is("ReconcilerStale")(state)) { + log("events stream not drained recently; running scripts/drain-workos-events.ts"); + runScript("drain", DRAIN_SCRIPT); + state = await readiness(); + log(describeMirrorReadiness(state)); + } + + if (!MirrorReadinessState.$is("Ready")(state)) { + console.error( + `[mirror-ready] the membership mirror is not ready: ${describeMirrorReadiness(state)}. ` + + "The deployed build would read membership from WorkOS on every request until it is. " + + "Check that WorkOS is reachable and the backfill has run, then rerun the deploy.", + ); + process.exit(1); + } + log("the membership mirror is ready"); +} finally { + await sql.end({ timeout: 5 }); +} diff --git a/apps/cloud/src/account/account-api.ts b/apps/cloud/src/account/account-api.ts index a8b631f170..37b4cb26eb 100644 --- a/apps/cloud/src/account/account-api.ts +++ b/apps/cloud/src/account/account-api.ts @@ -10,6 +10,7 @@ import { import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness } from "../auth/mirror-readiness"; import { WorkOsMirror } from "../auth/workos-mirror"; import { sessionFromSealed, type Session } from "../auth/middleware"; import { WorkOSClient } from "../auth/workos"; @@ -99,11 +100,15 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi * (the seat-gate) stays a residual requirement, satisfied by the app `boot`. */ export const workosAccountMiddleware = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer< + DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness + >, ) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer; export const makeAccountApiLive = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer< + DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness + >, ) => { // Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it // closes over the per-request postgres socket), so it can't be a self- diff --git a/apps/cloud/src/account/org-api-key-revoke.node.test.ts b/apps/cloud/src/account/org-api-key-revoke.node.test.ts index db322ba1bb..02228f3702 100644 --- a/apps/cloud/src/account/org-api-key-revoke.node.test.ts +++ b/apps/cloud/src/account/org-api-key-revoke.node.test.ts @@ -6,6 +6,7 @@ import { AccountError, AccountForbidden } from "@executor-js/api"; import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness"; import { ORG_SELECTOR_HEADER } from "../auth/organization"; import { WorkOSClient, type WorkOSClientService } from "../auth/workos"; import { WorkOsMirror } from "../auth/workos-mirror"; @@ -63,32 +64,12 @@ const session = (accountId: string) => ({ refreshedSession: null, }); -/** Membership roles: only ADMIN carries the `admin` role slug. */ +// Membership is read from the mirror, never from WorkOS: revoke makes no +// WorkOS call at all. const stubWorkOS = Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { - get: (_target, prop) => { - if (prop === "listUserMemberships") { - return (userId: string) => - Effect.succeed({ - data: [{ userId, organizationId: ORG, status: "active" }], - }); - } - if (prop === "getUserOrgMembership") { - return (organizationId: string, userId: string) => - Effect.succeed( - organizationId === ORG - ? { - id: `om_${userId}`, - userId, - organizationId, - role: { slug: userId === ADMIN ? "admin" : "member" }, - } - : null, - ); - } - return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); - }, + get: (_target, prop) => () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`), }), ); @@ -101,7 +82,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: org.id, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -110,7 +91,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ id, name: `Org ${id}`, slug: id, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -119,11 +100,12 @@ const stubUsers = Layer.succeed(UserStoreService)({ id: slug, name: `Org ${slug}`, slug, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, }), + markOrganizationDeleted: async () => null, deleteOrganizationCascade: async () => {}, }), ), @@ -147,12 +129,37 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ organizationBackfilledAt: () => Effect.die("revoke does not report seats"), }); -// Revoke lists no members either. +// The mirror as the directory reads it: both are active members of ORG, and +// only ADMIN carries the `admin` role. Revoke reads the caller's membership +// (the org check and the admin gate) and nothing else. +// The mirror is READY in these tests (backfill complete, reconciler caught +// up), so membership is read from the stubbed directory, never from WorkOS. +const stubReadiness = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + const stubDirectory = Layer.succeed(MemberDirectory)({ - membership: () => Effect.die("revoke does not read the member directory"), - members: () => Effect.die("revoke does not read the member directory"), - membersById: () => Effect.die("revoke does not read the member directory"), - findByEmail: () => Effect.die("revoke does not read the member directory"), + membership: (accountId, organizationId) => + Effect.succeed( + organizationId === ORG + ? { + accountId, + membershipId: `om_${accountId}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: accountId === ADMIN ? "admin" : "member", + status: "active" as const, + lastActiveAt: null, + } + : null, + ), + membershipById: () => Effect.die("revoke does not look up by membership id"), + membershipsOf: () => Effect.die("revoke does not list the caller's memberships"), + members: () => Effect.die("revoke does not list members"), + membersById: () => Effect.die("revoke does not batch members"), + findByEmail: () => Effect.die("revoke does not resolve emails"), }); const stubAutumn = Layer.succeed(AutumnService)({ @@ -194,6 +201,7 @@ const providerWith = (accountId: string) => { stubUsers, stubMirror, stubDirectory, + stubReadiness, stubApiKeys, stubAutumn, Layer.succeed(AccountCaller)({ session: session(accountId) }), diff --git a/apps/cloud/src/account/workos-account-service.ts b/apps/cloud/src/account/workos-account-service.ts index 5fadfb938f..e7abd95aa1 100644 --- a/apps/cloud/src/account/workos-account-service.ts +++ b/apps/cloud/src/account/workos-account-service.ts @@ -11,6 +11,7 @@ import { import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; import type { Session } from "../auth/middleware"; +import { MirrorReadiness } from "../auth/mirror-readiness"; import { WorkOSClient } from "../auth/workos"; import { ensureOrganizationBackfilled, mirrorInvitedMember } from "../auth/mirror-feeders"; import { WorkOsMirror, mirrorMembershipFromWorkOs } from "../auth/workos-mirror"; @@ -71,6 +72,7 @@ export const workosAccountProvider: Layer.Layer< | UserStoreService | WorkOsMirror | MemberDirectory + | MirrorReadiness | ApiKeyService | AutumnService | AccountCaller @@ -85,8 +87,8 @@ export const workosAccountProvider: Layer.Layer< // count read the change without waiting for the Events reconciler. const mirror = yield* WorkOsMirror; // Membership READS come from the mirror through the shared directory: the - // member list and the seat count are one local query each, never a - // WorkOS read per member. + // admin gate, the member list and the seat count are one local query + // each, never a WorkOS read. const directory = yield* MemberDirectory; // The caller, resolved once per request by the cookie-only session @@ -96,11 +98,17 @@ export const workosAccountProvider: Layer.Layer< const caller = yield* AccountCaller; // Capture the resolved service context once so the method bodies — which - // call `authorizeOrganization` (yields `WorkOSClient` + `UserStoreService`), - // the mirror feeders, and the seat reporter — can be erased to `R = never`, - // as the neutral AccountProvider shape requires. Provided per method below. + // call `authorizeOrganization` (yields `MemberDirectory` + `UserStoreService` + // + `WorkOSClient`), the mirror feeders, and the seat reporter — can be + // erased to `R = never`, as the neutral AccountProvider shape requires. + // Provided per method below. const ctx = yield* Effect.context< - WorkOSClient | UserStoreService | AutumnService | MemberDirectory | WorkOsMirror + | WorkOSClient + | UserStoreService + | AutumnService + | MemberDirectory + | MirrorReadiness + | WorkOsMirror >(); // Unauthenticated (missing/invalid session) => AccountUnauthorized, exactly @@ -115,10 +123,11 @@ export const workosAccountProvider: Layer.Layer< // org is a browser-global pinned to whichever org WorkOS last touched, so // falling back to it scopes a multi-org user's request to the WRONG org // (see workos-auth-provider.resolveSessionPrincipal). Membership is - // re-checked live, so the header is a selector, not a trust boundary — - // and two browser tabs on different orgs each send their own header, so + // re-checked against the mirror, so the header is a selector, not a trust + // boundary — and two browser tabs on different orgs each send their own header, so // they stay independent (see organization.ts). Yields the session + - // resolved org, or AccountNoOrganization. + // resolved org (carrying the caller's `memberRole` from that same + // membership read), or AccountNoOrganization. const requireOrganization = (headers: AccountHeaders) => Effect.gen(function* () { const session = yield* requireSession(); @@ -135,24 +144,28 @@ export const workosAccountProvider: Layer.Layer< }); // Mirror of org/handlers `requireAdmin`, but scoped to the resolved org. - const requireAdmin = (accountId: string, organizationId: string) => - Effect.gen(function* () { - const membership = yield* workos - .getUserOrgMembership(organizationId, accountId) - .pipe(Effect.catchTag("WorkOSError", toAccountError)); - if (!membership || membership.role?.slug !== "admin") { - return yield* new AccountForbidden(); - } - }); - - // Mirror of org/handlers `assertMembershipInSessionOrg` — ownership check so - // an admin can't mutate a membership id from another org. + // `authorizeOrganization` already read the caller's mirrored membership, + // required it to be ACTIVE, and normalized its role into `memberRole` — + // so the gate is that one value, not a second read of the same row. A + // pending admin invite is not an admin, and a member removed or demoted + // moments ago is denied as soon as the write-through or the Events + // reconciler has landed the change. + const requireAdmin = (org: { readonly memberRole: "admin" | "member" }) => + org.memberRole === "admin" ? Effect.void : Effect.fail(new AccountForbidden()); + + // Ownership check so an admin can't mutate a membership id from another + // org: the id must name a row the mirror holds for THIS org (any status — + // revoking a pending invite is a delete too). One point read on the + // membership id, scoped to the org: the member list the admin acted from + // is read from the same mirror, so every id it shows resolves here; a + // foreign or unknown id does not. A read failure is the same 500 as the + // admin gate's, never a refusal dressed up as "not yours". const assertMembershipInOrg = (organizationId: string, membershipId: string) => Effect.gen(function* () { - const membership = yield* workos - .getOrgMembership(membershipId) - .pipe(Effect.catchCause(() => Effect.succeed(null))); - if (!membership || membership.organizationId !== organizationId) { + const membership = yield* directory + .membershipById(organizationId, membershipId) + .pipe(Effect.catchTag("MemberDirectoryError", toAccountError)); + if (!membership) { return yield* new AccountForbidden(); } return membership; @@ -291,8 +304,8 @@ export const workosAccountProvider: Layer.Layer< // mint for themselves. listOrgApiKeys: (headers) => Effect.gen(function* () { - const { session, org } = yield* requireOrganization(headers); - yield* requireAdmin(session.accountId, org.id); + const { org } = yield* requireOrganization(headers); + yield* requireAdmin(org); const keys = yield* apiKeys .listOrgKeys({ organizationId: org.id }) .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError)); @@ -301,8 +314,8 @@ export const workosAccountProvider: Layer.Layer< createOrgApiKey: (headers, name) => Effect.gen(function* () { - const { session, org } = yield* requireOrganization(headers); - yield* requireAdmin(session.accountId, org.id); + const { org } = yield* requireOrganization(headers); + yield* requireAdmin(org); const trimmed = name.trim().slice(0, MAX_API_KEY_NAME_LENGTH); if (!trimmed) { return yield* new AccountError({ @@ -323,8 +336,8 @@ export const workosAccountProvider: Layer.Layer< // silent success and not a 500. revokeOrgApiKey: (headers, apiKeyId) => Effect.gen(function* () { - const { session, org } = yield* requireOrganization(headers); - yield* requireAdmin(session.accountId, org.id); + const { org } = yield* requireOrganization(headers); + yield* requireAdmin(org); yield* apiKeys.revokeOrgKey({ organizationId: org.id, keyId: apiKeyId }).pipe( Effect.catchTag("ApiKeyManagementError", toAccountError), Effect.catchTag("OrgApiKeyNotFound", () => @@ -383,8 +396,8 @@ export const workosAccountProvider: Layer.Layer< inviteMember: (headers, body) => Effect.gen(function* () { - const { session, org } = yield* requireOrganization(headers); - yield* requireAdmin(session.accountId, org.id); + const { org } = yield* requireOrganization(headers); + yield* requireAdmin(org); yield* reserveMemberSlot(org.id); const invitation = yield* workos .sendInvitation({ @@ -415,8 +428,8 @@ export const workosAccountProvider: Layer.Layer< removeMember: (headers, membershipId) => Effect.gen(function* () { - const { session, org } = yield* requireOrganization(headers); - yield* requireAdmin(session.accountId, org.id); + const { org } = yield* requireOrganization(headers); + yield* requireAdmin(org); const membership = yield* assertMembershipInOrg(org.id, membershipId); yield* workos .deleteOrgMembership(membershipId) @@ -426,18 +439,18 @@ export const workosAccountProvider: Layer.Layer< // delete — or a role change issued before it and delivered after // — is refused however it is stamped, while a replacement // membership WorkOS creates for the same member (a new id) is - // not. No WorkOS instant is in hand (WorkOS answers a delete with - // no time): the row keeps its own stamp, never the local clock, - // which read after WorkOS answered could post-date that - // replacement. + // not. No WorkOS instant is in hand (`null`: WorkOS answers a + // delete with no time, and the row was read from the mirror): the + // row keeps its own stamp, never the local clock, which read + // after WorkOS answered could post-date that replacement. yield* mirror .deleteMembership( { id: membershipId, - accountId: membership.userId, + accountId: membership.accountId, organizationId: membership.organizationId, }, - new Date(membership.updatedAt), + null, ) .pipe(Effect.catchTag("WorkOsMirrorError", toAccountError)); yield* forkReportMemberSeats(org.id).pipe(Effect.provideContext(ctx)); @@ -446,8 +459,8 @@ export const workosAccountProvider: Layer.Layer< updateMemberRole: (headers, membershipId, roleSlug) => Effect.gen(function* () { - const { session, org } = yield* requireOrganization(headers); - yield* requireAdmin(session.accountId, org.id); + const { org } = yield* requireOrganization(headers); + yield* requireAdmin(org); yield* assertMembershipInOrg(org.id, membershipId); const updated = yield* workos .updateOrgMembershipRole(membershipId, roleSlug) @@ -460,8 +473,8 @@ export const workosAccountProvider: Layer.Layer< updateOrgName: (headers, name) => Effect.gen(function* () { - const { session, org } = yield* requireOrganization(headers); - yield* requireAdmin(session.accountId, org.id); + const { org } = yield* requireOrganization(headers); + yield* requireAdmin(org); const updated = yield* workos .updateOrganization(org.id, name) .pipe(Effect.catchTag("WorkOSError", toAccountError)); diff --git a/apps/cloud/src/admin/admin-users-api.node.test.ts b/apps/cloud/src/admin/admin-users-api.node.test.ts new file mode 100644 index 0000000000..b29390be1a --- /dev/null +++ b/apps/cloud/src/admin/admin-users-api.node.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Layer } from "effect"; + +import { AdminUsersForbidden } from "@executor-js/api"; +import { MemberDirectory, type DirectoryMember } from "@executor-js/api/server"; + +import { ApiKeyService } from "../auth/api-keys"; +import { UserStoreService } from "../auth/context"; +import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness"; +import { ORG_SELECTOR_HEADER } from "../auth/organization"; +import { WorkOSClient, type WorkOSClientService } from "../auth/workos"; +import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror"; +import { authorizeTenant } from "./admin-users-api"; + +// --------------------------------------------------------------------------- +// The admin plane's SESSION credential: an admin member of the selected org, +// resolved against the membership mirror through the shared `MemberDirectory`. +// The org-key credential is pinned in `auth/org-api-key-auth.node.test.ts`; +// this file pins the session branch of `authorizeTenant`: +// - an ACTIVE `admin` membership yields the tenant id +// - an active plain member is refused +// - a pending admin invite is refused (not an admin until accepted) +// - no WorkOS call is made past session authentication +// --------------------------------------------------------------------------- + +const ORG = "org_tenant"; +const createdAt = new Date("2026-01-01T00:00:00.000Z"); + +const mirrored = ( + accountId: string, + overrides: Partial = {}, +): DirectoryMember => ({ + accountId, + membershipId: `om_${accountId}`, + organizationId: ORG, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active", + lastActiveAt: null, + ...overrides, +}); + +// The mirror as the directory reads it for ORG. +const memberships = new Map([ + ["user_admin", mirrored("user_admin", { role: "admin" })], + ["user_member", mirrored("user_member")], + ["user_invited_admin", mirrored("user_invited_admin", { role: "admin", status: "pending" })], +]); + +// The mirror is READY in these tests (backfill complete, reconciler caught +// up), so membership is read from the stubbed directory, never from WorkOS. +const stubReadiness = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: (accountId, organizationId) => + Effect.succeed(organizationId === ORG ? (memberships.get(accountId) ?? null) : null), + membershipById: () => Effect.die("tenant authorization does not look up by membership id"), + membershipsOf: () => Effect.die("tenant authorization reads one membership, not the list"), + members: () => Effect.die("tenant authorization does not list members"), + membersById: () => Effect.die("tenant authorization does not batch members"), + findByEmail: () => Effect.die("tenant authorization does not resolve emails"), +}); + +// No Authorization header in these tests: the api-key path falls through to +// the session path without validating anything. +const stubApiKeys = Layer.succeed(ApiKeyService)({ + validate: () => Effect.die("no bearer credential is presented"), + listUserKeys: () => Effect.die("tenant authorization does not list keys"), + createUserKey: () => Effect.die("tenant authorization does not create keys"), + revokeUserKey: () => Effect.die("tenant authorization does not revoke keys"), + listOrgKeys: () => Effect.die("tenant authorization does not list keys"), + createOrgKey: () => Effect.die("tenant authorization does not create keys"), + revokeOrgKey: () => Effect.die("tenant authorization does not revoke keys"), +}); + +// The mirror's account row as `ensureAccount` mints it: id only, profile +// columns unfilled until a WorkOS user payload arrives. +const bareAccount = (id: string) => ({ + id, + email: null, + firstName: null, + lastName: null, + avatarUrl: null, + workosUpdatedAt: null, + lastSignInAt: null, + createdAt, +}); + +// The selector is an org id, so only `getOrganization` is reached; the org +// row is already mirrored. +const stubUsers = Layer.succeed(UserStoreService)({ + use: (_op, fn) => + Effect.promise(() => + fn({ + ensureAccount: async (id: string) => bareAccount(id), + getAccount: async (id: string) => bareAccount(id), + upsertOrganization: async (org: { id: string; name: string }) => ({ + ...org, + slug: org.id, + backfilledAt: createdAt, + deletedAt: null, + workosUpdatedAt: null, + createdAt, + }), + getOrganization: async (id: string) => ({ + id, + name: `Org ${id}`, + slug: id, + backfilledAt: createdAt, + deletedAt: null, + workosUpdatedAt: null, + createdAt, + }), + getOrganizationBySlug: async (slug: string) => ({ + id: slug, + name: `Org ${slug}`, + slug, + backfilledAt: createdAt, + deletedAt: null, + workosUpdatedAt: null, + createdAt, + }), + markOrganizationDeleted: async () => null, + deleteOrganizationCascade: async () => {}, + }), + ), +}); + +// Authorization scans an organization the backfill never covered before it +// reads the mirror (`auth/organization.ts`); every org row above is marked +// backfilled, so the scan is never reached and the mirror is never written. +const stubMirror = Layer.succeed( + WorkOsMirror, + new Proxy({} as WorkOsMirrorShape, { + get: (_target, prop) => () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`), + }), +); + +// Only session authentication is served; membership is read from the mirror, +// so any other WorkOS call fails the test. +const stubWorkOS = (userId: string) => + Layer.succeed( + WorkOSClient, + new Proxy({} as WorkOSClientService, { + get: (_target, prop) => { + if (prop === "authenticateRequest") { + return () => + Effect.succeed({ userId, email: `${userId}@placeholder.test`, organizationId: null }); + } + return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); + }, + }), + ); + +const authorizeAs = (userId: string) => + authorizeTenant( + new Request("https://admin.invalid", { + headers: { cookie: "wos-session=sealed", [ORG_SELECTOR_HEADER]: ORG }, + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + stubDirectory, + stubApiKeys, + stubUsers, + stubWorkOS(userId), + stubMirror, + stubReadiness, + ), + ), + ); + +describe("authorizeTenant · admin session", () => { + it.effect("an active admin resolves the selected org as the tenant", () => + Effect.gen(function* () { + const tenant = yield* authorizeAs("user_admin"); + expect(tenant).toBe(ORG); + }), + ); + + it.effect("an active plain member is forbidden", () => + Effect.gen(function* () { + const error = yield* Effect.flip(authorizeAs("user_member")); + expect(error, "this plane serves the whole tenant; a member is not enough").toBeInstanceOf( + AdminUsersForbidden, + ); + }), + ); + + it.effect("a pending admin invite is forbidden", () => + Effect.gen(function* () { + const error = yield* Effect.flip(authorizeAs("user_invited_admin")); + expect(error, "an admin role that is still pending is not an admin").toBeInstanceOf( + AdminUsersForbidden, + ); + }), + ); +}); diff --git a/apps/cloud/src/admin/admin-users-api.ts b/apps/cloud/src/admin/admin-users-api.ts index 00c9a0e4d9..a6ade66a0a 100644 --- a/apps/cloud/src/admin/admin-users-api.ts +++ b/apps/cloud/src/admin/admin-users-api.ts @@ -8,8 +8,9 @@ // validated it and reported which org owns it, and there is no member // behind it to check membership for. This is the machine credential // (a customer's backend calling us). -// 2. an admin SESSION member -> the console. Requires a live `getUserOrgMembership` -// whose role slug is `admin` AND whose status is `active`, matching the +// 2. an admin SESSION member -> the console. Requires the caller's mirrored +// membership (the shared `MemberDirectory` over the local membership +// mirror) to carry the `admin` role AND `active` status, matching the // strictest existing cloud guard (`auth/handlers.ts`'s org-delete check) — // a pending admin invite is not an admin. // A plain member session, or a USER-scoped api key, is refused: both name one @@ -55,6 +56,8 @@ import type { Executor } from "@executor-js/sdk"; import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness } from "../auth/mirror-readiness"; +import { WorkOsMirror } from "../auth/workos-mirror"; import { isPlatformAuth, resolveBearerAuth } from "../auth/workos-auth-provider"; import { orgSelectorFromRequest, authorizeOrganizationSelector } from "../auth/organization"; import { WorkOSClient } from "../auth/workos"; @@ -67,13 +70,14 @@ import { CloudExecutionSeamsLayer } from "../engine/execution-stack"; * Returns only the organization id: nothing downstream needs to know WHICH of * the two credentials got the caller here, and keeping the acting member out of * the return value means no admin read can accidentally become subject-scoped. + * Exported for its test only. */ -const authorizeTenant = ( +export const authorizeTenant = ( request: Request, ): Effect.Effect< string, AdminUsersUnauthorized | AdminUsersForbidden, - WorkOSClient | ApiKeyService | UserStoreService + WorkOSClient | ApiKeyService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror > => Effect.gen(function* () { // (1) The bearer path. `resolveBearerAuth` (not `resolveApiKeyPrincipal`, @@ -92,7 +96,8 @@ const authorizeTenant = ( return yield* new AdminUsersForbidden(); } - // (2) The session path: a live admin membership in the selected org. + // (2) The session path: an active admin membership in the selected org, + // read from the mirror. const workos = yield* WorkOSClient; const session = yield* workos .authenticateRequest(request) @@ -101,20 +106,16 @@ const authorizeTenant = ( const selector = orgSelectorFromRequest(request) ?? session.organizationId; if (!selector) return yield* new AdminUsersForbidden(); - // Re-checks live membership, so the org selector header can only ever name - // an org the caller already belongs to. + // Re-checks membership against the mirror, so the org selector header can + // only ever name an org the caller already belongs to. That read requires + // an ACTIVE membership and reports its role as `memberRole`, so a pending + // admin invite never resolves and the admin gate is that one value — not + // a second read of the same row. const org = yield* authorizeOrganizationSelector(session.userId, selector).pipe( Effect.catchCause(() => Effect.succeed(null)), ); if (!org) return yield* new AdminUsersForbidden(); - - const membership = yield* workos - .getUserOrgMembership(org.id, session.userId) - .pipe(Effect.catchCause(() => Effect.succeed(null))); - // A pending admin invite is not an active admin — require both. - if (!membership || membership.status !== "active" || membership.role?.slug !== "admin") { - return yield* new AdminUsersForbidden(); - } + if (org.memberRole !== "admin") return yield* new AdminUsersForbidden(); return org.id; }); @@ -133,7 +134,15 @@ const withPlatformView = => Effect.gen(function* () { const organizationId = yield* authorizeTenant( @@ -169,13 +178,23 @@ export const workosAdminUsersProvider: Layer.Layer< | ApiKeyService | UserStoreService | MemberDirectory + | MirrorReadiness + | WorkOsMirror | DbProvider | PluginsProvider | HostConfig > = Layer.effect(AdminUsersProvider)( Effect.gen(function* () { const context = yield* Effect.context< - WorkOSClient | ApiKeyService | UserStoreService | DbProvider | PluginsProvider | HostConfig + | WorkOSClient + | ApiKeyService + | UserStoreService + | MemberDirectory + | MirrorReadiness + | WorkOsMirror + | DbProvider + | PluginsProvider + | HostConfig >(); const directory = yield* MemberDirectory; // The authorized tenant is what scopes the directory, so every read below @@ -246,7 +265,9 @@ const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUser * `/api` prefix as the rest of the cloud router. */ export const makeCloudAdminUsersRoutes = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer< + DbService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror + >, options: Parameters[1] = {}, ) => makeAdminUsersApiLayer( diff --git a/apps/cloud/src/api/layers.ts b/apps/cloud/src/api/layers.ts index f0875cc1dc..c693425995 100644 --- a/apps/cloud/src/api/layers.ts +++ b/apps/cloud/src/api/layers.ts @@ -11,6 +11,7 @@ import { import { SessionAuthLive } from "../auth/middleware-live"; import { UserStoreService } from "../auth/context"; import { cloudMemberDirectoryLayer } from "../auth/member-directory"; +import { MirrorReadiness } from "../auth/mirror-readiness"; import { WorkOsMirror } from "../auth/workos-mirror"; import { CloudAuthPublicHandlers, @@ -35,14 +36,23 @@ const WorkOsMirrorLive = WorkOsMirror.Live.pipe(Layer.provide(DbLive)); // The shared `MemberDirectory` read seam over the membership mirror — the // same per-request socket the mirror writes through. const MemberDirectoryLive = cloudMemberDirectoryLayer.pipe(Layer.provide(DbLive)); +// Whether the mirror may authorize this request at all (backfill complete, +// reconciler caught up) — read on the same socket before the membership row. +const MirrorReadinessLive = MirrorReadiness.Live.pipe(Layer.provide(DbLive)); // Per-request layer. Anything that opens an I/O object (postgres.js socket, // fetch stream readers, anything backed by a `Writable`) MUST live here — // `provideRequestScoped` rebuilds it per request so Cloudflare Workers' // I/O isolation is satisfied. See `api.request-scope.test.ts`. export const RequestScopedServicesLive: Layer.Layer< - DbService | UserStoreService | WorkOsMirror | MemberDirectory -> = Layer.mergeAll(DbLive, UserStoreLive, WorkOsMirrorLive, MemberDirectoryLive); + DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness +> = Layer.mergeAll( + DbLive, + UserStoreLive, + WorkOsMirrorLive, + MemberDirectoryLive, + MirrorReadinessLive, +); // Boot-scoped layer. Built once at worker boot, reused across requests. // Safe for config, in-memory caches, the global tracer provider, and @@ -67,7 +77,9 @@ export const BootSharedServices = Layer.mergeAll( // handler reads it for the free-organizations-per-user limit gate — one of the // few app-only billing touchpoints. (It is NOT on the neutral boot core.) export const makeNonProtectedApiLive = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer< + DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness + >, ) => HttpApiBuilder.layer(NonProtectedApi).pipe( Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)), @@ -82,7 +94,11 @@ export const makeNonProtectedApiLive = ( // the account and protected APIs. The `getDomainVerificationLink` handler also // gates on billing, so `AutumnService.Default` is provided here, not on the // neutral boot core. -export const makeOrgApiLive = (rsLive: Layer.Layer) => +export const makeOrgApiLive = ( + rsLive: Layer.Layer< + DbService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror + >, +) => HttpApiBuilder.layer(OrgHttpApi).pipe( Layer.provide(OrgHandlers), Layer.provide(orgAuthMiddleware(rsLive)), diff --git a/apps/cloud/src/api/protected-api-key-auth.node.test.ts b/apps/cloud/src/api/protected-api-key-auth.node.test.ts index 7be135a9e4..4aaf7cc1fb 100644 --- a/apps/cloud/src/api/protected-api-key-auth.node.test.ts +++ b/apps/cloud/src/api/protected-api-key-auth.node.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer } from "effect"; +import { MemberDirectory } from "@executor-js/api/server"; + import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness"; import { WorkOSClient, type WorkOSClientService } from "../auth/workos"; +import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror"; import { resolveProtectedPrincipal } from "./protected"; const createdAt = new Date("2026-01-01T00:00:00.000Z"); @@ -45,20 +49,43 @@ const stubWorkOS = Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { get: (_target, prop) => { - if (prop === "listUserMemberships") { - return (userId: string) => - Effect.succeed({ - data: - userId === "user_123" - ? [{ userId, organizationId: "org_123", status: "active" }] - : [], - }); - } return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); }, }), ); +// The mirror as the directory reads it: user_123 holds an active membership in +// org_123 and nothing else. Membership is never read from WorkOS. +// The mirror is READY in these tests (backfill complete, reconciler caught +// up), so membership is read from the stubbed directory, never from WorkOS. +const stubReadiness = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: (accountId, organizationId) => + Effect.succeed( + accountId === "user_123" && organizationId === "org_123" + ? { + accountId, + membershipId: `om_${accountId}_${organizationId}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active" as const, + lastActiveAt: null, + } + : null, + ), + membershipById: () => Effect.die("bearer resolution does not look up by membership id"), + membershipsOf: () => Effect.die("bearer resolution reads one membership, not the list"), + members: () => Effect.die("bearer resolution does not list members"), + membersById: () => Effect.die("bearer resolution does not batch members"), + findByEmail: () => Effect.die("bearer resolution does not resolve emails"), +}); + const stubUsers = Layer.succeed(UserStoreService)({ use: (_op, fn) => Effect.promise(() => @@ -68,7 +95,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: `org-slug-${org.id}`, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -77,7 +104,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ id, name: `Org ${id}`, slug: `org-slug-${id}`, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -86,19 +113,32 @@ const stubUsers = Layer.succeed(UserStoreService)({ id: "org_by_slug", name: `Org ${slug}`, slug, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, }), + markOrganizationDeleted: async () => null, deleteOrganizationCascade: async () => {}, }), ), }); +// Authorization scans an organization the backfill never covered before it +// reads the mirror (`auth/organization.ts`); every org row above is marked +// backfilled, so the scan is never reached and the mirror is never written. +const stubMirror = Layer.succeed( + WorkOsMirror, + new Proxy({} as WorkOsMirrorShape, { + get: (_target, prop) => () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`), + }), +); + const run = (request: Request) => resolveProtectedPrincipal(request).pipe( - Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers)), + Effect.provide( + Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness), + ), ); describe("protected API key auth", () => { diff --git a/apps/cloud/src/api/protected-jwt-auth.node.test.ts b/apps/cloud/src/api/protected-jwt-auth.node.test.ts index e4e2c041d5..02abb0b53e 100644 --- a/apps/cloud/src/api/protected-jwt-auth.node.test.ts +++ b/apps/cloud/src/api/protected-jwt-auth.node.test.ts @@ -2,10 +2,14 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer } from "effect"; import { SignJWT, createLocalJWKSet, exportJWK, generateKeyPair } from "jose"; +import { MemberDirectory } from "@executor-js/api/server"; + import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness"; import type { JwtBearerConfig } from "../auth/workos-auth-provider"; import { WorkOSClient, type WorkOSClientService } from "../auth/workos"; +import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror"; import { resolveProtectedPrincipal } from "./protected"; const createdAt = new Date("2026-01-01T00:00:00.000Z"); @@ -62,20 +66,43 @@ const stubWorkOS = Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { get: (_target, prop) => { - if (prop === "listUserMemberships") { - return (userId: string) => - Effect.succeed({ - data: - userId === "user_123" - ? [{ userId, organizationId: "org_123", status: "active" }] - : [], - }); - } return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); }, }), ); +// The mirror as the directory reads it: user_123 holds an active membership in +// org_123 and nothing else. Membership is never read from WorkOS. +// The mirror is READY in these tests (backfill complete, reconciler caught +// up), so membership is read from the stubbed directory, never from WorkOS. +const stubReadiness = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: (accountId, organizationId) => + Effect.succeed( + accountId === "user_123" && organizationId === "org_123" + ? { + accountId, + membershipId: `om_${accountId}_${organizationId}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active" as const, + lastActiveAt: null, + } + : null, + ), + membershipById: () => Effect.die("bearer resolution does not look up by membership id"), + membershipsOf: () => Effect.die("bearer resolution reads one membership, not the list"), + members: () => Effect.die("bearer resolution does not list members"), + membersById: () => Effect.die("bearer resolution does not batch members"), + findByEmail: () => Effect.die("bearer resolution does not resolve emails"), +}); + const stubUsers = Layer.succeed(UserStoreService)({ use: (_op, fn) => Effect.promise(() => @@ -85,7 +112,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: `org-slug-${org.id}`, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -94,7 +121,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ id, name: `Org ${id}`, slug: `org-slug-${id}`, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -103,19 +130,32 @@ const stubUsers = Layer.succeed(UserStoreService)({ id: "org_by_slug", name: `Org ${slug}`, slug, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, }), + markOrganizationDeleted: async () => null, deleteOrganizationCascade: async () => {}, }), ), }); +// Authorization scans an organization the backfill never covered before it +// reads the mirror (`auth/organization.ts`); every org row above is marked +// backfilled, so the scan is never reached and the mirror is never written. +const stubMirror = Layer.succeed( + WorkOsMirror, + new Proxy({} as WorkOsMirrorShape, { + get: (_target, prop) => () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`), + }), +); + const run = (request: Request, jwt: JwtBearerConfig) => resolveProtectedPrincipal(request, jwt).pipe( - Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers)), + Effect.provide( + Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness), + ), ); const request = (token: string) => diff --git a/apps/cloud/src/api/protected.ts b/apps/cloud/src/api/protected.ts index 417525d823..a9308025ef 100644 --- a/apps/cloud/src/api/protected.ts +++ b/apps/cloud/src/api/protected.ts @@ -10,11 +10,14 @@ import { requestScopedMiddleware, RouterConfigLive, type IdentityFailure, + type MemberDirectory, } from "@executor-js/api/server"; import { cloudPlugins, type CloudPlugins } from "../plugins"; import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness } from "../auth/mirror-readiness"; +import { WorkOsMirror } from "../auth/workos-mirror"; import { cloudIdentityFailureStrategy, workosIdentityLayer } from "../auth/workos-auth-provider"; import { AutumnService } from "../extensions/billing/service"; import { DbService } from "../db/db"; @@ -32,8 +35,8 @@ export { // One `HttpRouter` middleware that: // 1. resolves identity via the NEUTRAL `IdentityProvider` (api-key BEATS sealed -// session, decided INSIDE cloud's `workosIdentityLayer`), verifying live org -// membership, +// session, decided INSIDE cloud's `workosIdentityLayer`), verifying org +// membership against the local mirror, // 2. builds the per-request executor + engine, // 3. provides `AuthContext` + the execution-stack services to the handler. // @@ -93,9 +96,13 @@ const ExecutionStackMiddleware = makeExecutionStackMiddleware< // executor plane that meters, not to the neutral boot core. (`/autumn`, the // account seat-gate, and the createOrganization free-limit gate each provide it // where they run.) -export const makeProtectedApiLive = (rsLive: Layer.Layer) => { +export const makeProtectedApiLive = ( + rsLive: Layer.Layer< + DbService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror + >, +) => { // The neutral `IdentityProvider`, built per request: it reads `UserStoreService` - // from `rsLive` and the WorkOS control plane (`WorkOSClient` + `ApiKeyService`, + // + `MemberDirectory` from `rsLive` and the WorkOS control plane (`WorkOSClient` + `ApiKeyService`, // stateless config — no per-request I/O socket) for the org-resolution path. // `orDie` because a WorkOS config error is unrecoverable. const identityLive = workosIdentityLayer.pipe( diff --git a/apps/cloud/src/api/router.ts b/apps/cloud/src/api/router.ts index 8c80825ef2..7bb7da2479 100644 --- a/apps/cloud/src/api/router.ts +++ b/apps/cloud/src/api/router.ts @@ -8,6 +8,7 @@ import { } from "@executor-js/api/server"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness } from "../auth/mirror-readiness"; import { WorkOsMirror } from "../auth/workos-mirror"; import { DbService } from "../db/db"; import { makeAccountApiLive } from "../account/account-api"; @@ -35,7 +36,9 @@ import { makeProtectedApiLive } from "./protected"; // assert per-request semantics — see // `apps/cloud/src/api.request-scope.node.test.ts`. export const makeApiLive = ( - requestScopedLive: Layer.Layer, + requestScopedLive: Layer.Layer< + DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness + >, ) => { const BillingRoutesLive = AutumnRoutesLive.pipe( Layer.provide(requestScopedMiddleware(requestScopedLive).layer), diff --git a/apps/cloud/src/auth/api.ts b/apps/cloud/src/auth/api.ts index 3f69c0ce6c..4ff1fc25e1 100644 --- a/apps/cloud/src/auth/api.ts +++ b/apps/cloud/src/auth/api.ts @@ -1,7 +1,7 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; import { Schema } from "effect"; import { UserStoreError, WorkOSError, WorkOsMirrorError } from "./errors"; -import { NoOrganization } from "@executor-js/api/server"; +import { MemberDirectoryError, NoOrganization } from "@executor-js/api/server"; import { SessionAuth } from "./middleware"; const AuthUser = Schema.Struct({ @@ -166,6 +166,16 @@ export class OrganizationDeletionForbidden extends Schema.TaggedErrorClass()( + "OrganizationDeletionIncomplete", + { step: Schema.Literals(["billing"]) }, + { httpApiStatus: 500 }, +) {} + export const AUTH_PATHS = { login: "/api/auth/login", logout: "/api/auth/logout", @@ -173,8 +183,10 @@ export const AUTH_PATHS = { } as const; // The login callback and the org handlers feed the membership mirror, so a -// mirror write failure is one of their wire errors (same 500 as a store failure). -const AuthErrors = [UserStoreError, WorkOSError, WorkOsMirrorError] as const; +// mirror write failure is one of their wire errors (same 500 as a store +// failure); the session handlers READ it (membership, the org list, the admin +// gate), so a directory read failure is one too. +const AuthErrors = [UserStoreError, WorkOSError, WorkOsMirrorError, MemberDirectoryError] as const; const McpApprovalErrors = [ NoOrganization, McpExecutionNotFoundError, @@ -216,7 +228,7 @@ export class CloudAuthApi extends HttpApiGroup.make("cloudAuth") .add( HttpApiEndpoint.get("organizations", "/auth/organizations", { success: AuthOrganizationsResponse, - error: WorkOSError, + error: [WorkOSError, UserStoreError, MemberDirectoryError], }), ) .add( @@ -230,7 +242,12 @@ export class CloudAuthApi extends HttpApiGroup.make("cloudAuth") HttpApiEndpoint.post("deleteOrganization", "/auth/delete-organization", { payload: DeleteOrganizationBody, success: DeleteOrganizationResponse, - error: [...AuthErrors, NoOrganization, OrganizationDeletionForbidden], + error: [ + ...AuthErrors, + NoOrganization, + OrganizationDeletionForbidden, + OrganizationDeletionIncomplete, + ], }), ) .add( diff --git a/apps/cloud/src/auth/doc-gate.ts b/apps/cloud/src/auth/doc-gate.ts index 16d6145bc9..cc33730c4a 100644 --- a/apps/cloud/src/auth/doc-gate.ts +++ b/apps/cloud/src/auth/doc-gate.ts @@ -41,6 +41,9 @@ import { makeDbLayer } from "../db/db"; import { makeUserStoreLayer, UserStoreService } from "./context"; import { parseCookie } from "./cookies"; import { LAST_ORG_COOKIE } from "./last-org-cookie"; +import { makeMemberDirectoryLayer } from "./member-directory"; +import { makeMirrorReadinessLayer } from "./mirror-readiness"; +import { makeWorkOsMirrorLayer } from "./workos-mirror"; import { sealedSessionDisplayName } from "./middleware"; import { authorizeOrganizationSelector } from "./organization"; import { loginPath, safeReturnTo } from "./return-to"; @@ -164,18 +167,27 @@ const organizationDisplay = async ( : { name: "", slug: "" }; }; -// Live membership check for the last-org cookie's slug. Same authorize path -// as any org selector — the cookie is a preference, so a slug the user can't -// access (stale after removal/deletion, or forged) resolves to null and the -// bare path falls through to today's canonicalize-onto-session-org behavior. -// Per-request store layers for the same reason as organizationDisplay. +// Membership check (against the local mirror) for the last-org cookie's slug. +// Same authorize path as any org selector — the cookie is a preference, so a +// slug the user can't access (stale after removal/deletion, or forged) resolves +// to null and the bare path falls through to today's +// canonicalize-onto-session-org behavior. Per-request store layers for the +// same reason as organizationDisplay; both stores share the one socket. const authorizeLastOrgSlug = async ( userId: string, slug: string, ): Promise<{ readonly id: string } | null> => { + const dbLive = makeDbLayer(); const exit = await getRuntime().runPromiseExit( authorizeOrganizationSelector(userId, slug).pipe( - Effect.provide(Layer.provide(makeUserStoreLayer(), makeDbLayer())), + Effect.provide( + Layer.mergeAll( + makeUserStoreLayer(), + makeMemberDirectoryLayer(), + makeMirrorReadinessLayer(), + makeWorkOsMirrorLayer(), + ).pipe(Layer.provide(dbLive)), + ), ), ); return Exit.isSuccess(exit) ? exit.value : null; @@ -257,7 +269,7 @@ export const authGateMiddleware = createMiddleware({ type: "request" }).server( // contract is untouched because an unknown-but-valid slug in the URL reads // as slugged, not bare. When the cookie matches the session's own org (the // overwhelmingly common single-org case) the client-side OrgSlugGate - // already canonicalizes onto it, so skip the live membership check and the + // already canonicalizes onto it, so skip the membership check and the // redirect entirely. const lastOrgSlug = parseCookie(cookieHeader, LAST_ORG_COOKIE); const firstSegment = pathname.split("/")[1] ?? ""; diff --git a/apps/cloud/src/auth/handlers.ts b/apps/cloud/src/auth/handlers.ts index e7184f253a..6453a0547f 100644 --- a/apps/cloud/src/auth/handlers.ts +++ b/apps/cloud/src/auth/handlers.ts @@ -10,8 +10,9 @@ import { McpExecutionNotFoundError, McpSessionForbiddenError, OrganizationDeletionForbidden, + OrganizationDeletionIncomplete, } from "./api"; -import { NoOrganization } from "@executor-js/api/server"; +import { MemberDirectory, NoOrganization } from "@executor-js/api/server"; // Pure constants/codec module (no React) — safe in the backend graph. import { AUTH_HINT_COOKIE } from "@executor-js/react/multiplayer/auth-hint"; import { SessionContext, SessionCookies } from "./middleware"; @@ -22,7 +23,7 @@ import { mirrorMembership, mirrorSignIn } from "./mirror-feeders"; import { env } from "cloudflare:workers"; import { WorkOSError } from "./errors"; import { WorkOSClient } from "./workos"; -import { AutumnService } from "../extensions/billing/service"; +import { AutumnService, autumnStatusOf } from "../extensions/billing/service"; import { forkReportMemberSeats } from "../extensions/billing/member-seats"; import { captureCauseEffect } from "../observability"; import { @@ -35,7 +36,9 @@ import { ORG_SELECTOR_HEADER, authorizeOrganization, authorizeOrganizationSelector, + markOrganizationDeleted, resolveOrganization, + type AuthorizeOrganizationOptions, } from "./organization"; import { mcpSessionStub } from "@executor-js/cloudflare/mcp/session-stub"; @@ -103,27 +106,30 @@ const firstPathSegment = (path: string): string | null => { const requestedOrgSelectorFromReturnTo = (returnTo: string): string | null => firstPathSegment(returnTo); -const requireSelectedOrganization = Effect.gen(function* () { - const session = yield* SessionContext; - const headers = yield* requestHeaders; - const selector = headers[ORG_SELECTOR_HEADER] ?? session.organizationId; - if (!selector) { - return yield* new NoOrganization(); - } - - const org = yield* authorizeOrganizationSelector(session.accountId, selector).pipe( - Effect.catch(() => Effect.fail(new NoOrganization())), - ); - if (!org) { - return yield* new NoOrganization(); - } - - return { - ...session, - organizationId: org.id, - memberRole: org.memberRole, - }; -}); +const selectedOrganization = (options: AuthorizeOrganizationOptions = {}) => + Effect.gen(function* () { + const session = yield* SessionContext; + const headers = yield* requestHeaders; + const selector = headers[ORG_SELECTOR_HEADER] ?? session.organizationId; + if (!selector) { + return yield* new NoOrganization(); + } + + const org = yield* authorizeOrganizationSelector(session.accountId, selector, options).pipe( + Effect.catch(() => Effect.fail(new NoOrganization())), + ); + if (!org) { + return yield* new NoOrganization(); + } + + return { + ...session, + organizationId: org.id, + memberRole: org.memberRole, + }; + }); + +const requireSelectedOrganization = selectedOrganization(); const getMcpSessionStub = (mcpSessionId: string) => mcpSessionStub(env.MCP_SESSION, mcpSessionId); @@ -397,20 +403,29 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( ) .handle("organizations", () => Effect.gen(function* () { - const workos = yield* WorkOSClient; + const directory = yield* MemberDirectory; const session = yield* SessionContext; - const memberships = yield* workos.listUserMemberships(session.accountId); + // The caller's memberships (active + pending, as WorkOS listed them + // before) from the local mirror — one indexed read, no WorkOS call. + const memberships = yield* directory.membershipsOf(session.accountId); // Resolve through the mirror (not WorkOS directly) so each org's // URL slug is minted/read — the switcher navigates to `/`. + // An org marked deleted (its deletion is in progress or failed + // part-way, see deleteOrganization) refuses every session, so it + // is not a place the switcher can go. const organizations = yield* Effect.all( - memberships.data.map((m) => + memberships.map((m) => resolveOrganization(m.organizationId).pipe( - Effect.map((org) => ({ - id: org.id, - name: org.name, - slug: org.slug, - })), + Effect.map((org) => + org.deletedAt === null + ? { + id: org.id, + name: org.name, + slug: org.slug, + } + : null, + ), Effect.orElseSucceed(() => null), ), ), @@ -431,10 +446,10 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( const autumn = yield* AutumnService; const name = payload.name.trim(); - const memberships = yield* workos.listUserMemberships(session.accountId); - const activeMemberships = memberships.data.filter( - (membership) => membership.status === "active", - ); + // The free-organizations-per-user limit counts the caller's ACTIVE + // memberships, read from the local mirror. + const directory = yield* MemberDirectory; + const activeMemberships = yield* directory.membershipsOf(session.accountId, ["active"]); if (isOverFreeOrganizationLimit(activeMemberships)) { const paidOrganizationIds = yield* Effect.all( @@ -536,15 +551,18 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( // Target the caller's currently-selected org (honors the org-selector // header, same as the other org-scoped auth handlers). NoOrganization - // when the session has no org to act on. - const session = yield* requireSelectedOrganization; + // when the session has no org to act on. An org already MARKED + // deleted still resolves here — and only here — so an admin whose + // earlier attempt failed after the mark can send it again and finish. + const session = yield* selectedOrganization({ deleted: "allow" }); const organizationId = session.organizationId; - // Admin-only. Live WorkOS check so a member removed/demoted moments - // ago can't delete the workspace. A pending admin invite is not an - // active admin, so require active status too. - const membership = yield* workos.getUserOrgMembership(organizationId, session.accountId); - if (!membership || membership.status !== "active" || membership.role?.slug !== "admin") { + // Admin-only. `requireSelectedOrganization` already read the caller's + // mirrored membership, required it ACTIVE (a pending admin invite is + // not an admin) and reported its role, so the gate is that one + // value: a member removed or demoted moments ago is denied once the + // write-through or the Events reconciler has landed the change. + if (session.memberRole !== "admin") { return yield* new OrganizationDeletionForbidden(); } @@ -555,20 +573,86 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( return yield* new OrganizationDeletionForbidden(); } - // WorkOS FIRST. Once the org is gone there, membership authorization - // fails for every member, so the workspace is truly deleted even if a - // later local step lags (leftover local rows become unreachable, not - // user-visible). The reverse order risks the org resurrecting as an - // empty workspace when a later request re-mirrors it with a new slug. - yield* workos.deleteOrganization(organizationId); - - // Purge all local tenant data, secrets, and the org's memberships in - // one transaction, leaving the org row as a tombstone marked deleted - // (so a login that fetched its membership list before the deletion - // cannot re-mint the org afterwards). If this fails after the WorkOS - // delete already succeeded, the org is gone for everyone - // (unreachable) but its secrets/tenant rows linger orphaned — alert - // loudly so that window gets swept, then surface the failure. + // Four steps, each idempotent, so a request that failed part-way + // can be sent again and finish the job. The local purge is the LAST + // step that can fail: it removes the org's membership rows — the + // admin's own among them, the row that admits the retry above — so + // nothing that can fail may run after it, or the retry it needs + // would be refused at the door. And the WorkOS delete comes AFTER + // billing: it is the one step that makes the org unrecoverable + // outside this database, so nothing that can fail runs between it + // and the purge except the purge itself — a billing failure leaves + // the WorkOS org intact, the memberships still live there, and the + // retry admitted by WorkOS and mirror alike. + // + // 1. Mark the org deleted LOCALLY. Membership is authorized from the + // local mirror (`authorizeOrganization`), not from WorkOS, so + // this — not the WorkOS delete — is what revokes every member's + // access, and it happens before anything that can fail leaves + // the org half-deleted. From here on every session is refused + // at once, whether or not the steps below land. + yield* markOrganizationDeleted(organizationId); + + // 2. Cancel billing. A 404 — "no such customer" — is a retry after + // this step landed (or an org that was never provisioned): + // nothing to cancel, and not a failure. Matched on the status, + // not Autumn's `customer_not_found` code, because the delete + // endpoint answers an unknown customer with a bare 404 (and the + // Autumn emulator serves no delete route at all). Any other + // Autumn failure surfaces as an incomplete deletion: the WorkOS + // delete and the purge below must not run until billing is + // cancelled, because after them the admin can no longer send + // the request again. + yield* autumn + .use((client) => client.customers.delete({ customerId: organizationId })) + .pipe( + Effect.catchIf( + (failure) => + Predicate.isTagged(failure, "AutumnCustomerNotFoundError") || + autumnStatusOf(failure) === 404, + () => + Effect.logInfo( + "deleteOrganization: Autumn has no customer for the org; nothing to cancel", + { organizationId }, + ), + ), + Effect.tapError((error) => + Effect.logError( + "deleteOrganization: org marked deleted but the Autumn customer could not be deleted; retry the deletion", + { organizationId, error }, + ), + ), + Effect.mapError(() => new OrganizationDeletionIncomplete({ step: "billing" })), + ); + + // 3. Delete the WorkOS org (cascades its memberships, invitations, + // and domains there). "Already deleted" (404) is a retry after + // the purge failed, not a failure: fall through. + yield* workos + .deleteOrganization(organizationId) + .pipe( + Effect.catchTag("WorkOSError", (error) => + error.status === 404 + ? Effect.logInfo( + "deleteOrganization: WorkOS org already deleted; finishing the deletion", + { organizationId }, + ) + : Effect.fail(error), + ), + ); + + // 4. Purge all local tenant data, secrets, and the org's memberships + // in one transaction, keeping the org row as a tombstone marked + // deleted (step 1's mark stands; a login that fetched its + // membership list before the deletion cannot re-mint the org + // afterwards). If this fails, the org is already unreachable + // (step 1) but its secrets/tenant rows linger — alert loudly, + // surface the failure, and the admin retries: the transaction + // rolled back, so their membership row still admits them (read + // from the mirror even while it is not ready — WorkOS no longer + // lists the org's members); step 1 keeps its mark, and steps 2 + // and 3 tolerate the gone customer and org, so the retry reaches + // this purge again. const deletedAt = new Date(yield* Clock.currentTimeMillis); yield* users .use("deleteOrganizationCascade", (s) => @@ -577,28 +661,12 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( .pipe( Effect.tapError((error) => Effect.logError( - "deleteOrganization: WorkOS org deleted but local purge failed, tenant data and secrets orphaned", + "deleteOrganization: org marked deleted, removed from WorkOS and Autumn, but local purge failed, tenant data and secrets orphaned; retry the deletion", { organizationId, error }, ), ), ); - // Cancel billing. Best-effort: the org is already deleted, so a - // lingering Autumn customer is a billing loose end (log loudly) rather - // than a correctness failure that should 500 the caller. - yield* autumn - .use((client) => client.customers.delete({ customerId: organizationId })) - .pipe( - // Includes the "customer never existed" answer: nothing to cancel - // is a fine outcome for a deleted org, and it is still worth a line. - Effect.catch((error) => - Effect.logWarning("deleteOrganization: failed to delete Autumn customer", { - organizationId, - error, - }), - ), - ); - // The caller's session is pinned to the now-deleted org — clear it so // the browser bounces to login and rehydrates to another membership // (or the create-org screen when they have none left). diff --git a/apps/cloud/src/auth/last-org-cookie.ts b/apps/cloud/src/auth/last-org-cookie.ts index ef6242604f..897afe2212 100644 --- a/apps/cloud/src/auth/last-org-cookie.ts +++ b/apps/cloud/src/auth/last-org-cookie.ts @@ -12,7 +12,7 @@ // - the login callback prefers it when picking the org for a fresh session // with a bare returnTo (handlers.ts) // -// It is a PREFERENCE, never an authority: both readers re-check live membership +// It is a PREFERENCE, never an authority: both readers re-check membership // through the same authorize path as any org selector, so a stale or forged // value at worst falls back to today's behavior. Not HttpOnly — the client is // the writer. Deliberately NOT cleared on logout: surviving the session is what diff --git a/apps/cloud/src/auth/member-directory.ts b/apps/cloud/src/auth/member-directory.ts index 01174d88f3..d5fedb38a2 100644 --- a/apps/cloud/src/auth/member-directory.ts +++ b/apps/cloud/src/auth/member-directory.ts @@ -141,6 +141,32 @@ const makeService = (db: DrizzleDb): MemberDirectoryShape => { return row === undefined ? null : toMember(row); }), + // The unique index on `membership_id` makes this a point read; the org + // predicate is what refuses an id that belongs to another org. + membershipById: (organizationId, membershipId) => + read("membershipById", async () => { + const rows = await select() + .where( + and( + eq(memberships.organizationId, organizationId), + eq(memberships.membershipId, membershipId), + ), + ) + .limit(1); + const row = rows[0]; + return row === undefined ? null : toMember(row); + }), + + membershipsOf: (accountId, statuses = DEFAULT_MEMBER_STATUSES) => + read("membershipsOf", async () => { + const rows = await select() + .where( + and(eq(memberships.accountId, accountId), inArray(memberships.status, statuses), known), + ) + .orderBy(asc(memberships.organizationId)); + return toMembers(rows); + }), + members, membersById: (organizationId, accountIds, statuses = DEFAULT_MEMBER_STATUSES) => @@ -179,3 +205,13 @@ const makeService = (db: DrizzleDb): MemberDirectoryShape => { /** The cloud `MemberDirectory` over the per-request `DbService`. */ export const cloudMemberDirectoryLayer: Layer.Layer = Layer.effect(MemberDirectory)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db))); + +/** + * A FRESH `MemberDirectory` layer (new layer value per call), for a service + * built once but invoked across many Workers requests — the MCP + * org-authorization seam and the document gate — for the same reason + * `makeUserStoreLayer` exists: a memoized const layer would pin the first + * request's postgres socket. See [[makeDbLayer]]. + */ +export const makeMemberDirectoryLayer = (): Layer.Layer => + Layer.effect(MemberDirectory)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db))); diff --git a/apps/cloud/src/auth/mirror-feeders.node.test.ts b/apps/cloud/src/auth/mirror-feeders.node.test.ts index a344f4553c..5834d156aa 100644 --- a/apps/cloud/src/auth/mirror-feeders.node.test.ts +++ b/apps/cloud/src/auth/mirror-feeders.node.test.ts @@ -20,6 +20,19 @@ // so a replay of the membership as it was before the delete cannot // restore it while a replacement WorkOS created meanwhile is accepted // - `updateMemberRole` writes the role WorkOS returned +// - deleting an org marks it deleted locally FIRST, so every member's +// session is refused at once even when the billing cancel, the WorkOS +// delete, or the local purge fails afterwards; billing is cancelled +// BEFORE the WorkOS delete, so a failed cancel leaves the WorkOS org +// intact and the retry finishes the deletion; a retry after WorkOS +// already deleted the org still runs the purge — even while the mirror +// is not ready, when WorkOS can no longer vouch for the admin; a marked +// org leaves the switcher +// - authorization scans an organization the backfill never covered (its +// `backfilled_at` is missing) from WorkOS before reading its mirror, +// once, so a member the mirror never recorded is admitted; an +// organization the mirror does not hold at all is resolved from WorkOS +// for a caller WorkOS confirms as its member, and minted for nobody else // - the seat gate trusts the mirror's count only for an organization whose // membership list was scanned from WorkOS in full: an unmarked one is // scanned first (once), so a partial mirror never admits an invite past @@ -64,17 +77,22 @@ import { AccountCaller, workosAccountProvider } from "../account/workos-account- import { RequestScopedServicesLive } from "../api/layers"; import { DbService } from "../db/db"; import { forkReportMemberSeats } from "../extensions/billing/member-seats"; -import { AutumnService } from "../extensions/billing/service"; +import { AutumnError, AutumnService, type AutumnFailure } from "../extensions/billing/service"; import { ApiKeyService } from "./api-keys"; import { UserStoreService } from "./context"; -import { WorkOSError } from "./errors"; +import { UserStoreError, WorkOSError } from "./errors"; import { CloudAuthPublicHandlers, CloudSessionAuthHandlers, NonProtectedApi } from "./handlers"; import { LAST_ORG_COOKIE } from "./last-org-cookie"; import { encodeLoginState } from "./login-state"; import { cloudMemberDirectoryLayer } from "./member-directory"; import { SessionAuthLive } from "./middleware-live"; import { mirrorSignIn } from "./mirror-feeders"; -import { ORG_SELECTOR_HEADER } from "./organization"; +import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness"; +import { + ORG_SELECTOR_HEADER, + authorizeOrganization, + markOrganizationDeleted, +} from "./organization"; import { WorkOSClient, type WorkOSClientService } from "./workos"; import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror"; import { backfillOrganization, backfillWorkOsMirror } from "./workos-mirror-backfill"; @@ -154,6 +172,14 @@ const seedOrganization = (id: string) => ), ); +// The mirror is READY throughout (backfill complete, reconciler caught up): +// every membership read below is against the mirror, never WorkOS. The +// readiness rule itself is pinned in workos-mirror.node.test.ts and the +// fallback in org-selector-auth.node.test.ts. +const readyMirror = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + const stubAutumn = Layer.succeed(AutumnService)({ use: () => Effect.die("feeders do not read billing"), ensureCustomer: () => Effect.void, @@ -540,6 +566,534 @@ describe("a delayed sign-in feeder", () => { }); }); +describe("session handlers read membership from the mirror", () => { + /** + * The session routes over the live request-scoped services. `workos` adds + * to the fake WorkOS (only session authentication by default: every + * membership read against WorkOS dies); `services` replaces the per-request + * layer, so a test can fail one store call on purpose. + */ + const sessionHandler = ( + userId: string, + options: { + readonly workos?: Partial; + readonly services?: Layer.Layer< + DbService | UserStoreService | WorkOsMirror | MemberDirectory + >; + readonly autumn?: Layer.Layer; + /** The mirror's readiness for this request; ready unless a test says otherwise. */ + readonly readiness?: Layer.Layer; + } = {}, + ) => + HttpRouter.toWebHandler( + HttpApiBuilder.layer(NonProtectedApi).pipe( + Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)), + Layer.provide( + requestScopedMiddleware( + Layer.mergeAll( + options.services ?? RequestScopedServicesLive, + options.readiness ?? readyMirror, + ), + ).layer, + ), + Layer.provideMerge(SessionAuthLive), + Layer.provideMerge(options.autumn ?? stubAutumn), + Layer.provideMerge( + stubWorkOS({ + ...options.workos, + authenticateSealedSession: () => + Effect.succeed({ + userId, + email: `${userId}@placeholder.test`, + organizationId: null, + } as never), + }), + ), + Layer.provideMerge(HttpServer.layerServices), + Layer.provideMerge(RouterConfigLive), + ), + { disableLogger: true }, + ).handler; + + /** The org row as the mirror holds it, or null once purged. */ + const readOrganization = (org: string) => + Effect.runPromise( + Effect.flatMap(UserStoreService.asEffect(), (users) => + users.use("getOrganization", (s) => s.getOrganization(org)), + ).pipe( + Effect.provide(UserStoreService.Live.pipe(Layer.provide(DbService.Live))), + Effect.scoped, + ), + ); + + /** + * `authorizeOrganization` over the live stores and a READY mirror, as every + * protected request runs it; `workos` serves whatever the check may read + * from WorkOS (nothing, by default: any read dies). + */ + const authorize = ( + userId: string, + org: string, + workos: Layer.Layer = stubWorkOS({}), + ) => + Effect.runPromise( + authorizeOrganization(userId, org).pipe( + Effect.provide( + Layer.mergeAll( + UserStoreService.Live, + WorkOsMirror.Live, + cloudMemberDirectoryLayer, + readyMirror, + ).pipe(Layer.provideMerge(DbService.Live)), + ), + Effect.provide(workos), + Effect.scoped, + ), + ); + + /** Whether `userId` is authorized for `org` right now. */ + const authorized = async (userId: string, org: string) => (await authorize(userId, org)) !== null; + + /** A request-scoped layer whose `deleteOrganizationCascade` fails, everything else live. */ + const servicesWithFailingPurge = (purges: string[]) => + Layer.mergeAll( + Layer.effect(UserStoreService)( + Effect.map(UserStoreService.asEffect(), (live): UserStoreService["Service"] => ({ + use: (op, fn) => + op === "deleteOrganizationCascade" + ? Effect.sync(() => { + purges.push(op); + }).pipe( + Effect.flatMap(() => + Effect.fail(new UserStoreError({ operation: op, reason: "connection_closed" })), + ), + ) + : live.use(op, fn), + })), + ).pipe(Layer.provide(UserStoreService.Live)), + WorkOsMirror.Live, + cloudMemberDirectoryLayer, + ).pipe(Layer.provideMerge(DbService.Live)); + + const deletingAutumn = Layer.succeed(AutumnService)({ + use: () => Effect.succeed({} as never), + ensureCustomer: () => Effect.void, + checkExecutionBalance: () => Effect.die("deletion does not check balances"), + trackExecution: () => Effect.void, + setMemberSeats: () => Effect.void, + }); + + /** + * Mirror `org` — marked as scanned (an empty listing at T1), as the one-off + * backfill leaves every org, so authorization reads its mirror without a + * WorkOS scan — and `userId`'s membership in it; returns the org's slug. + */ + const seedMembership = async ( + userId: string, + org: string, + status: "active" | "pending", + role: "admin" | "member" = "member", + ) => { + const slug = await seedOrganization(org); + await Effect.runPromise( + Effect.flatMap(WorkOsMirror.asEffect(), (mirror) => + Effect.andThen( + mirror.applyOrganizationScan({ + organizationId: org, + listedAt: new Date(T1), + members: [], + }), + mirror.upsertMembership({ + id: `om_${userId}_${org}`, + accountId: userId, + organizationId: org, + role, + status, + updatedAt: new Date(T1), + }), + ), + ).pipe(Effect.provide(WorkOsMirror.Live.pipe(Layer.provide(DbService.Live))), Effect.scoped), + ); + return slug; + }; + + const deleteOrganizationRequest = (org: string) => + new Request("http://test.local/auth/delete-organization", { + method: "POST", + headers: { + cookie: "wos-session=sealed", + "content-type": "application/json", + [ORG_SELECTOR_HEADER]: org, + }, + body: JSON.stringify({ confirmName: `Org ${org}` }), + }); + + it("lists the caller's organizations from the mirror, with their slugs", async () => { + const userId = freshId("user"); + const activeOrg = freshId("org"); + const pendingOrg = freshId("org"); + const otherUser = freshId("user"); + const foreignOrg = freshId("org"); + const activeSlug = await seedMembership(userId, activeOrg, "active"); + const pendingSlug = await seedMembership(userId, pendingOrg, "pending"); + await seedMembership(otherUser, foreignOrg, "active"); + + const response = await sessionHandler(userId)( + new Request("http://test.local/auth/organizations", { + headers: { cookie: "wos-session=sealed" }, + }), + ); + + expect(response.status).toBe(200); + const body = (await response.json()) as { + organizations: { id: string; slug: string }[]; + activeOrganizationId: string | null; + }; + expect( + body.organizations.map((o) => [o.id, o.slug]).sort(), + "active and pending memberships, each with the mirror's slug; nobody else's", + ).toEqual( + [ + [activeOrg, activeSlug], + [pendingOrg, pendingSlug], + ].sort(), + ); + expect(body.activeOrganizationId).toBeNull(); + }); + + it("refuses to delete an org for a pending admin, before WorkOS is asked", async () => { + const userId = freshId("user"); + const org = freshId("org"); + // An admin role that is still pending: the org gate reads the mirror and + // requires an ACTIVE membership, so the invite grants no deletion right. + await seedMembership(userId, org, "pending", "admin"); + + const response = await sessionHandler(userId)(deleteOrganizationRequest(org)); + + // The selector resolves no active membership, so the request fails at the + // org check (NoOrganization) — the handler never reaches the WorkOS + // delete, which the stub would die on. + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ _tag: "NoOrganization" }); + }); + + it("refuses to delete an org for an active plain member, before WorkOS is asked", async () => { + const userId = freshId("user"); + const org = freshId("org"); + await seedMembership(userId, org, "active", "member"); + + const response = await sessionHandler(userId)(deleteOrganizationRequest(org)); + + expect(response.status).toBe(403); + expect( + await response.json(), + "an active member who is not an admin may not delete the org", + ).toMatchObject({ _tag: "OrganizationDeletionForbidden" }); + }); + + it("revokes every member's access the moment deletion starts, even when the local purge fails, and finishes on a retry after WorkOS already deleted the org", async () => { + const admin = freshId("user"); + const member = freshId("user"); + const org = freshId("org"); + await seedMembership(admin, org, "active", "admin"); + await seedMembership(member, org, "active", "member"); + expect(await authorized(member, org), "live before the deletion").toBe(true); + + // First attempt: WorkOS deletes the org, then the local purge fails. + const workosDeletes: string[] = []; + const purges: string[] = []; + const failing = sessionHandler(admin, { + services: servicesWithFailingPurge(purges), + autumn: deletingAutumn, + workos: { + deleteOrganization: (organizationId) => + Effect.sync(() => { + workosDeletes.push(organizationId); + }), + }, + }); + const first = await failing(deleteOrganizationRequest(org)); + expect(first.status, "the failed purge is surfaced, not hidden").toBe(500); + expect(workosDeletes).toEqual([org]); + expect(purges).toEqual(["deleteOrganizationCascade"]); + expect( + (await readOrganization(org))?.deletedAt, + "the org was marked deleted BEFORE WorkOS was asked", + ).not.toBeNull(); + // Membership rows are still there (the purge did not run), yet nobody + // is authorized: the mark, not the WorkOS delete, revokes access. + expect(await authorized(member, org)).toBe(false); + expect(await authorized(admin, org)).toBe(false); + + // Retry: WorkOS now answers "already deleted"; the local purge completes. + const retry = sessionHandler(admin, { + autumn: deletingAutumn, + workos: { + deleteOrganization: () => Effect.fail(new WorkOSError({ status: 404 })), + }, + }); + const second = await retry(deleteOrganizationRequest(org)); + expect(second.status, "the admin's own membership still admits the retry").toBe(200); + expect(await second.json()).toEqual({ success: true }); + expect( + (await readOrganization(org))?.deletedAt, + "the org row stays as a tombstone, marked deleted", + ).not.toBeNull(); + expect(await readMembers(org), "its memberships are purged").toEqual([]); + expect(await authorized(admin, org)).toBe(false); + }); + + it("finishes on a retry after the billing cancel failed, and only purges once billing is cancelled", async () => { + const admin = freshId("user"); + const member = freshId("user"); + const org = freshId("org"); + await seedMembership(admin, org, "active", "admin"); + await seedMembership(member, org, "active", "member"); + + // Autumn is down for the first attempt; on the retry it answers "no such + // customer" — the first attempt's cancel may have landed after all, or + // the org was never provisioned — which is nothing to cancel. The delete + // endpoint says so with a bare 404 (no `customer_not_found` code), so + // that is the shape the retry gets: an `AutumnError` whose SDK cause + // carries the status. + let billingCalls = 0; + const flakyAutumn = Layer.succeed(AutumnService)({ + use: () => + Effect.suspend(() => { + billingCalls += 1; + const failure: AutumnFailure = + billingCalls === 1 + ? new AutumnError({ message: "Autumn SDK request failed" }) + : new AutumnError({ + message: "Autumn SDK request failed", + cause: { statusCode: 404, body: '{"message":"Not Found"}' }, + }); + return Effect.fail(failure); + }), + ensureCustomer: () => Effect.void, + checkExecutionBalance: () => Effect.die("deletion does not check balances"), + trackExecution: () => Effect.void, + setMemberSeats: () => Effect.void, + }); + const workosDeletes: string[] = []; + const handler = sessionHandler(admin, { + autumn: flakyAutumn, + workos: { + deleteOrganization: (organizationId) => + Effect.sync(() => { + workosDeletes.push(organizationId); + }), + }, + }); + + const first = await handler(deleteOrganizationRequest(org)); + expect(first.status, "the failed billing cancel is surfaced, not hidden").toBe(500); + expect(await first.json()).toMatchObject({ + _tag: "OrganizationDeletionIncomplete", + step: "billing", + }); + expect(workosDeletes, "the WorkOS org is NOT deleted before billing is cancelled").toEqual([]); + expect(billingCalls).toBe(1); + expect((await readOrganization(org))?.deletedAt, "the org is marked deleted").not.toBeNull(); + expect( + (await readMembers(org)).map((m) => m.accountId).sort(), + "the purge did NOT run: the membership rows are still there", + ).toEqual([admin, member].sort()); + expect(await authorized(member, org), "yet nobody is authorized: the mark stands").toBe(false); + + const second = await handler(deleteOrganizationRequest(org)); + expect(second.status, "the admin's own membership row still admits the retry").toBe(200); + expect(await second.json()).toEqual({ success: true }); + expect(workosDeletes, "WorkOS is asked once billing is cancelled").toEqual([org]); + expect(billingCalls, "billing is asked again and tolerates the gone customer").toBe(2); + expect(await readMembers(org), "and the purge ran: its memberships are gone").toEqual([]); + expect( + (await readOrganization(org))?.deletedAt, + "the org row stays as a tombstone", + ).not.toBeNull(); + expect(await authorized(admin, org)).toBe(false); + }); + + it("finishes on a retry while the mirror is not ready, after WorkOS already deleted the org", async () => { + const admin = freshId("user"); + const member = freshId("user"); + const org = freshId("org"); + await seedMembership(admin, org, "active", "admin"); + await seedMembership(member, org, "active", "member"); + + // First attempt: billing cancelled, WorkOS org deleted, local purge fails. + const purges: string[] = []; + const first = await sessionHandler(admin, { + services: servicesWithFailingPurge(purges), + autumn: deletingAutumn, + workos: { deleteOrganization: () => Effect.void }, + })(deleteOrganizationRequest(org)); + expect(first.status).toBe(500); + expect(purges).toEqual(["deleteOrganizationCascade"]); + + // The reconciler stalls before the retry. WorkOS no longer lists the org + // or the admin's membership in it — and the fallback must not ask it: + // the stub dies on `listUserMemberships`. The admin's own mirror row, + // which the failed purge left behind, is what admits the retry. + const retry = sessionHandler(admin, { + autumn: deletingAutumn, + readiness: Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.ReconcilerStale({ drainedAt: null })), + }), + workos: { deleteOrganization: () => Effect.fail(new WorkOSError({ status: 404 })) }, + }); + const second = await retry(deleteOrganizationRequest(org)); + expect(second.status, "the retry is admitted from the mirror, not WorkOS").toBe(200); + expect(await second.json()).toEqual({ success: true }); + expect(await readMembers(org), "and the purge ran").toEqual([]); + expect((await readOrganization(org))?.deletedAt).not.toBeNull(); + }); + + it("resolves an organization the mirror does not hold from WorkOS for its member, and mints it for nobody else", async () => { + const memberId = freshId("user"); + const outsider = freshId("user"); + const org = freshId("org"); + // Never seeded: the org predates the mirror and nobody has signed in to + // it since — a CLI token names it, and the JWT path has no login feeder. + const calls: string[] = []; + const workos = stubWorkOS({ + getUserOrgMembership: (organizationId, userId) => { + calls.push(`getUserOrgMembership:${userId}`); + return Effect.succeed( + userId === memberId + ? (workosMembership(userId, organizationId, { role: { slug: "admin" } }) as never) + : null, + ); + }, + getOrganization: (id) => { + calls.push(`getOrganization:${id}`); + return Effect.succeed({ + object: "organization", + id, + name: "Pre-mirror Org", + allowProfilesOutsideOrganization: false, + domains: [], + createdAt: T1, + updatedAt: T1, + externalId: null, + metadata: {}, + } as never); + }, + listOrgMembers: (organizationId) => { + calls.push(`listOrgMembers:${organizationId}`); + return Effect.succeed({ + object: "list" as const, + data: [workosMembership(memberId, org, { role: { slug: "admin" } })] as never[], + listMetadata: { before: null, after: null }, + }); + }, + getUser: (id) => { + calls.push(`getUser:${id}`); + return Effect.succeed(workosUser(id) as never); + }, + }); + + // A non-member first: WorkOS is asked for THEIR membership only, and + // nothing is minted. + expect(await authorize(outsider, org, workos)).toBeNull(); + expect(calls).toEqual([`getUserOrgMembership:${outsider}`]); + expect(await readOrganization(org), "no row for an org the caller is not in").toBeNull(); + + // The member: WorkOS confirms the membership, the org is minted and + // scanned once, and the caller is authorized from the scan's result. + const first = await authorize(memberId, org, workos); + expect(first?.memberRole).toBe("admin"); + expect(first?.name).toBe("Pre-mirror Org"); + expect(calls.slice(1)).toEqual([ + `getUserOrgMembership:${memberId}`, + `getOrganization:${org}`, + `listOrgMembers:${org}`, + `getUser:${memberId}`, + ]); + expect((await readMembers(org)).map((m) => m.accountId)).toEqual([memberId]); + + // Now held and marked: the next check reads the mirror alone. + const second = await authorize(memberId, org, workos); + expect(second?.id).toBe(org); + expect(calls, "no further WorkOS read").toHaveLength(5); + }); + + it("scans an organization the backfill never covered before authorizing from its mirror, once", async () => { + const userId = freshId("user"); + const outsider = freshId("user"); + const org = freshId("org"); + // The org row exists (mirrored lazily, or by another member's login) but + // was never scanned, and holds no membership rows at all: the caller is + // a WorkOS member the mirror has never recorded. + await seedOrganization(org); + const calls: string[] = []; + const workos = stubWorkOS({ + listOrgMembers: (organizationId, statuses) => { + calls.push(`listOrgMembers:${organizationId}`); + expect(statuses, "the scan lists every status").toEqual(["active", "pending", "inactive"]); + return Effect.succeed({ + object: "list" as const, + data: [workosMembership(userId, org, { role: { slug: "admin" } })] as never[], + listMetadata: { before: null, after: null }, + }); + }, + getUser: (id) => { + calls.push(`getUser:${id}`); + return Effect.succeed(workosUser(id) as never); + }, + }); + + const first = await authorize(userId, org, workos); + expect(first?.memberRole, "authorized from the scan's result, with the scanned role").toBe( + "admin", + ); + expect(calls, "one scan: the listing and one getUser per member").toEqual([ + `listOrgMembers:${org}`, + `getUser:${userId}`, + ]); + expect( + (await readMembers(org)).map((m) => m.accountId), + "the scan filled the mirror", + ).toEqual([userId]); + + const second = await authorize(userId, org, workos); + expect(second?.id).toBe(org); + expect(calls, "the org is now marked: the second check reads the mirror alone").toEqual([ + `listOrgMembers:${org}`, + `getUser:${userId}`, + ]); + expect( + await authorize(outsider, org, workos), + "a non-member is refused from the mirror", + ).toBeNull(); + expect(calls, "without a scan").toHaveLength(2); + }); + + it("keeps a marked org out of the organization switcher", async () => { + const userId = freshId("user"); + const live = freshId("org"); + const marked = freshId("org"); + const liveSlug = await seedMembership(userId, live, "active"); + await seedMembership(userId, marked, "active"); + await Effect.runPromise( + markOrganizationDeleted(marked).pipe( + Effect.provide(UserStoreService.Live.pipe(Layer.provide(DbService.Live))), + Effect.scoped, + ), + ); + + const response = await sessionHandler(userId)( + new Request("http://test.local/auth/organizations", { + headers: { cookie: "wos-session=sealed" }, + }), + ); + + expect(response.status).toBe(200); + const body = (await response.json()) as { organizations: { id: string; slug: string }[] }; + expect(body.organizations.map((o) => [o.id, o.slug])).toEqual([[live, liveSlug]]); + }); +}); + describe("account service writes through to the mirror", () => { const ADMIN = freshId("user"); const TARGET = freshId("user"); @@ -565,11 +1119,13 @@ describe("account service writes through to the mirror", () => { }); /** - * The provider layer over the LIVE mirror + user store (test db) and a fake - * WorkOS in which ADMIN administers `org` and TARGET is a plain member. - * `deleted` records the WorkOS-side deletes so "WorkOS first" is assertable. - * Provided around the WHOLE test body so the postgres socket outlives the - * provider call under test. + * The provider layer over the LIVE mirror + user store + directory (test db) + * and a fake WorkOS that only serves the WRITES. Membership reads — the org + * check, the admin gate, the ownership check on the target — come from the + * mirror, so `seedTarget` mirrors ADMIN as the org's admin alongside TARGET; + * any membership READ against WorkOS dies. `deleted` records the WorkOS-side + * deletes so "WorkOS first" is assertable. Provided around the WHOLE test + * body so the postgres socket outlives the provider call under test. */ const providerLayer = ( org: string, @@ -579,23 +1135,8 @@ describe("account service writes through to the mirror", () => { readonly autumn?: Layer.Layer; } = {}, ) => { - const list = (data: readonly unknown[]) => - Effect.succeed({ - object: "list" as const, - data: data as never[], - listMetadata: { before: null, after: null }, - }); const workos = stubWorkOS({ ...options.workos, - listUserMemberships: (userId) => list([workosMembership(userId, org)]), - getUserOrgMembership: (organizationId, userId) => - Effect.succeed( - workosMembership(userId, organizationId, { - role: { slug: userId === ADMIN ? "admin" : "member" }, - }) as never, - ), - getOrgMembership: (membershipId) => - Effect.succeed(workosMembership(TARGET, org, { id: membershipId }) as never), deleteOrgMembership: (membershipId) => Effect.sync(() => { deleted.push(membershipId); @@ -615,6 +1156,7 @@ describe("account service writes through to the mirror", () => { UserStoreService.Live, WorkOsMirror.Live, cloudMemberDirectoryLayer, + readyMirror, ); return workosAccountProvider.pipe( Layer.provide( @@ -630,10 +1172,11 @@ describe("account service writes through to the mirror", () => { ); }; - // TARGET as an existing member of `org`, seeded through the live mirror. - // The org is marked backfilled (as the one-off backfill leaves every org) - // unless a test wants the unscanned state, so a seat count reads the mirror - // rather than scanning WorkOS. + // ADMIN as the org's admin and TARGET as an existing member of `org`, + // seeded through the live mirror — the rows the provider's membership reads + // resolve against. The org is marked backfilled (as the one-off backfill + // leaves every org) unless a test wants the unscanned state, so a seat + // count reads the mirror rather than scanning WorkOS. const seedTarget = ( org: string, options: { readonly backfilled: boolean } = { backfilled: true }, @@ -648,6 +1191,14 @@ describe("account service writes through to the mirror", () => { updatedAt: new Date(T1), }), ); + yield* mirror.upsertMembership({ + id: `om_${ADMIN}_${org}`, + accountId: ADMIN, + organizationId: org, + role: "admin", + status: "active", + updatedAt: new Date(T1), + }); yield* mirror.upsertMembership({ id: `om_${TARGET}_${org}`, accountId: TARGET, @@ -761,9 +1312,9 @@ describe("account service writes through to the mirror", () => { () => { const org = freshId("org"); const listed: string[] = []; - // A free plan (limit 3). The mirror holds ONE member of the org (TARGET) - // and the org is unmarked; WorkOS lists three. Only a count taken after - // the scan refuses the invite. + // A free plan (limit 3). The mirror holds TWO members of the org (ADMIN, + // TARGET) and the org is unmarked; WorkOS lists four. Only a count + // taken after the scan refuses the invite. const freeAutumn = Layer.succeed(AutumnService)({ use: () => Effect.succeed({ subscriptions: [] } as never), ensureCustomer: () => Effect.void, @@ -790,7 +1341,11 @@ describe("account service writes through to the mirror", () => { ]); return Effect.succeed({ object: "list" as const, - data: [TARGET, ...others].map((userId) => workosMembership(userId, org)) as never[], + data: [ADMIN, TARGET, ...others].map((userId) => + workosMembership(userId, org, { + role: { slug: userId === ADMIN ? "admin" : "member" }, + }), + ) as never[], listMetadata: { before: null, after: null }, }); }, @@ -816,7 +1371,7 @@ describe("account service writes through to the mirror", () => { expect( (yield* membersOf(org)).map((m) => m.accountId).sort(), "and the scan filled the mirror", - ).toEqual([TARGET, ...others].sort()); + ).toEqual([ADMIN, TARGET, ...others].sort()); const again = yield* invite(); expect(again).toBeInstanceOf(AccountForbidden); @@ -892,6 +1447,28 @@ describe("account service writes through to the mirror", () => { expect(members.find((m) => m.accountId === TARGET)?.role).toBe("admin"); }).pipe(Effect.provide(providerLayer(org, []))); }); + + it.effect("removeMember refuses a membership id the org does not hold, before WorkOS", () => { + const org = freshId("org"); + const other = freshId("org"); + const deleted: string[] = []; + return Effect.gen(function* () { + yield* seedTarget(org); + const account = yield* AccountProvider; + + // A membership id from ANOTHER org (leaked, guessed) is not in this + // org's mirror, so the ownership check refuses it and nothing is + // deleted anywhere. + const error = yield* Effect.flip( + account.removeMember({ [ORG_SELECTOR_HEADER]: org }, `om_${TARGET}_${other}`), + ); + + expect(error).toBeInstanceOf(AccountForbidden); + expect(deleted, "the gate runs BEFORE the WorkOS delete").toEqual([]); + const members = yield* membersOf(org); + expect(members.map((m) => m.accountId).sort()).toEqual([ADMIN, TARGET].sort()); + }).pipe(Effect.provide(providerLayer(org, deleted))); + }); }); describe("seat reporter", () => { @@ -933,6 +1510,8 @@ describe("seat reporter", () => { const directoryWith = (org: string, active: number) => Layer.succeed(MemberDirectory)({ membership: () => Effect.die("the seat reporter lists, it does not look up"), + membershipById: () => Effect.die("the seat reporter lists, it does not look up"), + membershipsOf: () => Effect.die("the seat reporter lists, it does not look up"), membersById: () => Effect.die("the seat reporter lists, it does not look up"), findByEmail: () => Effect.die("the seat reporter lists, it does not look up"), members: (organizationId, query) => { diff --git a/apps/cloud/src/auth/mirror-readiness-store.ts b/apps/cloud/src/auth/mirror-readiness-store.ts new file mode 100644 index 0000000000..c2e65524cd --- /dev/null +++ b/apps/cloud/src/auth/mirror-readiness-store.ts @@ -0,0 +1,116 @@ +// --------------------------------------------------------------------------- +// Mirror READINESS: whether the local membership mirror may be trusted as +// the membership authority for a request, or WorkOS must still be asked. +// +// The mirror is fed by login, write-through, and the Events API reconciler +// (`workos-mirror.ts`), and is complete only once the one-off backfill has +// written every organization and the reconciler has caught up to the +// present. Before that, two things go wrong if it is trusted anyway: +// - a member who has not signed in since the mirror shipped has no row +// yet, and every protected request of theirs is refused — the backfill +// is what writes them; +// - a member revoked in the WorkOS dashboard while the reconciler was not +// running still holds an active row, and keeps their access until the +// stream is replayed — the reconciler is what tombstones them. +// So readiness is BOTH: the backfill's completion mark +// (`workos_sync.backfill_completed_at`, written once by a run that covered +// every live organization) AND a recent drain of the events stream +// (`workos_sync.drained_at`, moved forward by every reconciler run that read +// the stream to its end). The lag budget bounds how far behind the reconciler +// may be: it runs every minute, so a mark older than the budget means it has +// stalled (WorkOS unreachable, the cron not deployed, a backlog draining over +// many runs) and the mirror may be missing revocations. While either half is +// missing the authorization path reads membership from WorkOS instead +// (`organization.ts`), exactly as it did before the cutover; nothing is +// denied or granted on the mirror's word. +// +// The rule and the row read live here, free of `cloudflare:workers`, so the +// deploy gate (`scripts/ensure-workos-mirror-ready.ts`) applies the SAME rule +// over a plain postgres.js connection under bun before the build that trusts +// the mirror goes live. The request-scoped service is `mirror-readiness.ts`. +// --------------------------------------------------------------------------- + +import { eq } from "drizzle-orm"; +import { Data, Duration } from "effect"; + +import type { DrizzleDb } from "../db/db"; +import { workosSync } from "../db/schema"; +import { WORKOS_EVENTS_STREAM_ID } from "./workos-mirror-store"; + +/** + * How far behind the present the reconciler's last drain may be before the + * mirror stops being trusted. The reconciler runs every minute and a healthy + * run drains in one tick; ten minutes absorbs a few missed ticks and a short + * WorkOS blip without falling back, and bounds how long a dashboard-side + * revocation could go unseen if it did. + */ +export const MIRROR_RECONCILER_LAG_BUDGET = Duration.minutes(10); + +/** + * What the readiness check found. `Ready` is the only state in which the + * mirror authorizes; the other two name which half is missing so the fallback + * can be logged with its cause. + */ +export type MirrorReadinessState = Data.TaggedEnum<{ + readonly Ready: {}; + /** No backfill run has covered every organization yet. */ + readonly BackfillPending: {}; + /** The backfill is done but the reconciler has not drained within the budget (`drainedAt` null = never). */ + readonly ReconcilerStale: { readonly drainedAt: Date | null }; +}>; +export const MirrorReadinessState = Data.taggedEnum(); + +/** The two `workos_sync` columns the rule reads, as the events row holds them (or no row at all). */ +export interface MirrorReadinessRow { + readonly backfillCompletedAt: Date | null; + readonly drainedAt: Date | null; +} + +/** + * The readiness rule over the events row as of `now`: ready when the + * backfill has completed AND the last drain is within + * {@link MIRROR_RECONCILER_LAG_BUDGET} of `now`. A missing row is a mirror + * that was never backfilled. Pure, so the deploy gate and the request path + * cannot disagree. + */ +export const mirrorReadinessFrom = ( + row: MirrorReadinessRow | null, + now: Date, +): MirrorReadinessState => { + if (row === null || row.backfillCompletedAt === null) + return MirrorReadinessState.BackfillPending(); + const drainedAt = row.drainedAt; + if ( + drainedAt === null || + now.getTime() - drainedAt.getTime() > Duration.toMillis(MIRROR_RECONCILER_LAG_BUDGET) + ) { + return MirrorReadinessState.ReconcilerStale({ drainedAt }); + } + return MirrorReadinessState.Ready(); +}; + +/** Read the events row's readiness columns and apply {@link mirrorReadinessFrom} as of `now`. */ +export const readMirrorReadiness = async ( + db: DrizzleDb, + now: Date, +): Promise => { + const rows = await db + .select({ + backfillCompletedAt: workosSync.backfillCompletedAt, + drainedAt: workosSync.drainedAt, + }) + .from(workosSync) + .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID)); + return mirrorReadinessFrom(rows[0] ?? null, now); +}; + +/** One line naming the state, for logs and the deploy gate; never carries member data. */ +export const describeMirrorReadiness = (state: MirrorReadinessState): string => + MirrorReadinessState.$match(state, { + Ready: () => "ready", + BackfillPending: () => "backfill pending: no backfill run has covered every organization yet", + ReconcilerStale: ({ drainedAt }) => + drainedAt === null + ? "reconciler stale: the events reconciler has never drained the stream" + : `reconciler stale: the events stream was last drained at ${drainedAt.toISOString()}, past the ${Duration.format(MIRROR_RECONCILER_LAG_BUDGET)} budget`, + }); diff --git a/apps/cloud/src/auth/mirror-readiness.ts b/apps/cloud/src/auth/mirror-readiness.ts new file mode 100644 index 0000000000..1bf8ac05de --- /dev/null +++ b/apps/cloud/src/auth/mirror-readiness.ts @@ -0,0 +1,69 @@ +// --------------------------------------------------------------------------- +// MirrorReadiness — the request-scoped service that answers whether the +// membership mirror may authorize this request (see +// `mirror-readiness-store.ts` for the rule and why it exists). +// +// Per-request layer shape, like `UserStoreService` and `WorkOsMirror`: it +// reads the request's postgres socket, so it is rebuilt per request +// (`RequestScopedServicesLive`) and never shared across Workers requests. One +// indexed point read per authorization, on the same socket the membership +// read uses next. +// --------------------------------------------------------------------------- + +import { Clock, Context, Effect, Layer } from "effect"; + +import { DbService, type DrizzleDb } from "../db/db"; +import { + WorkOsMirrorError, + tryPromiseService, + userStoreReasonFromCause, + withServiceLogging, +} from "./errors"; +import { readMirrorReadiness, type MirrorReadinessState } from "./mirror-readiness-store"; + +export { + MIRROR_RECONCILER_LAG_BUDGET, + MirrorReadinessState, + describeMirrorReadiness, + mirrorReadinessFrom, + type MirrorReadinessRow, +} from "./mirror-readiness-store"; + +export interface MirrorReadinessShape { + /** + * The mirror's readiness as of now. Fails with `WorkOsMirrorError` when the + * row cannot be read — the caller must not treat that as either ready or + * not; it is the same infra failure as any other mirror read. + */ + readonly state: () => Effect.Effect; +} + +const makeService = (db: DrizzleDb): MirrorReadinessShape => ({ + state: () => + Effect.flatMap(Clock.currentTimeMillis, (millis) => + withServiceLogging( + "workos_mirror.readiness", + (failure) => + new WorkOsMirrorError({ + operation: "readiness", + reason: userStoreReasonFromCause(failure), + }), + tryPromiseService(() => readMirrorReadiness(db, new Date(millis))), + ), + ), +}); + +export class MirrorReadiness extends Context.Service()( + "@executor-js/cloud/MirrorReadiness", +) { + static Live = Layer.effect(this)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db))); +} + +/** + * A FRESH `MirrorReadiness` layer (new layer value per call), for a service + * built once but invoked across many Workers requests — the MCP + * org-authorization seam and the document gate — for the same reason + * `makeUserStoreLayer` exists. See [[makeDbLayer]]. + */ +export const makeMirrorReadinessLayer = (): Layer.Layer => + Layer.effect(MirrorReadiness)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db))); diff --git a/apps/cloud/src/auth/org-api-key-auth.node.test.ts b/apps/cloud/src/auth/org-api-key-auth.node.test.ts index 14e2006f55..5358e75994 100644 --- a/apps/cloud/src/auth/org-api-key-auth.node.test.ts +++ b/apps/cloud/src/auth/org-api-key-auth.node.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from "@effect/vitest"; -import { Effect, Layer } from "effect"; +import { Cause, Effect, Exit, Layer } from "effect"; + +import { MemberDirectory, NoOrganization } from "@executor-js/api/server"; import { ApiKeyService } from "./api-keys"; import { UserStoreService } from "./context"; +import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness"; import { WorkOSClient, type WorkOSClientService } from "./workos"; +import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror"; import { isPlatformAuth, resolveApiKeyPrincipal, resolveBearerAuth } from "./workos-auth-provider"; // Groundwork for the PRIVILEGED, org-level API key: it resolves to the platform @@ -58,22 +62,45 @@ const stubWorkOS = Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { get: (_target, prop) => { - if (prop === "listUserMemberships") { - return (userId: string) => - Effect.succeed({ - data: - userId === "user_123" - ? [{ userId, organizationId: "org_123", status: "active" }] - : [], - }); - } - // An org key must NOT trigger a membership check — there is no user to - // check. Any such call dies here, which is the assertion. + // Membership is read from the mirror, never from WorkOS; any WorkOS call + // dies here. return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); }, }), ); +// The mirror as the directory reads it: user_123 holds an active membership in +// org_123 and nothing else. Membership is never read from WorkOS. +// The mirror is READY in these tests (backfill complete, reconciler caught +// up), so membership is read from the stubbed directory, never from WorkOS. +const stubReadiness = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: (accountId, organizationId) => + Effect.succeed( + accountId === "user_123" && organizationId === "org_123" + ? { + accountId, + membershipId: `om_${accountId}_${organizationId}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active" as const, + lastActiveAt: null, + } + : null, + ), + membershipById: () => Effect.die("bearer resolution does not look up by membership id"), + membershipsOf: () => Effect.die("bearer resolution reads one membership, not the list"), + members: () => Effect.die("bearer resolution does not list members"), + membersById: () => Effect.die("bearer resolution does not batch members"), + findByEmail: () => Effect.die("bearer resolution does not resolve emails"), +}); + const stubUsers = Layer.succeed(UserStoreService)({ use: (_op, fn) => Effect.promise(() => @@ -83,7 +110,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: `org-slug-${org.id}`, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -92,7 +119,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ id, name: `Org ${id}`, slug: `org-slug-${id}`, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -101,17 +128,35 @@ const stubUsers = Layer.succeed(UserStoreService)({ id: "org_by_slug", name: `Org ${slug}`, slug, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, }), + markOrganizationDeleted: async () => null, deleteOrganizationCascade: async () => {}, }), ), }); -const layers = Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers); +// Authorization scans an organization the backfill never covered before it +// reads the mirror (`auth/organization.ts`); every org row above is marked +// backfilled, so the scan is never reached and the mirror is never written. +const stubMirror = Layer.succeed( + WorkOsMirror, + new Proxy({} as WorkOsMirrorShape, { + get: (_target, prop) => () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`), + }), +); + +const layers = Layer.mergeAll( + stubApiKeys, + stubWorkOS, + stubUsers, + stubDirectory, + stubMirror, + stubReadiness, +); const bearer = (token: string) => new Request("https://executor.test/api/tools", { @@ -137,6 +182,67 @@ describe("org-level API keys", () => { }), ); + it.effect("are refused once the org is marked deleted", () => + Effect.gen(function* () { + const deletedOrgUsers = Layer.succeed(UserStoreService)({ + use: (_op, fn) => + Effect.promise(() => + fn({ + ensureAccount: async (id: string) => bareAccount(id), + getAccount: async (id: string) => bareAccount(id), + upsertOrganization: async (org: { id: string; name: string }) => ({ + ...org, + slug: `org-slug-${org.id}`, + backfilledAt: createdAt, + deletedAt: createdAt, + workosUpdatedAt: null, + createdAt, + }), + getOrganization: async (id: string) => ({ + id, + name: `Org ${id}`, + slug: `org-slug-${id}`, + backfilledAt: createdAt, + deletedAt: createdAt, + workosUpdatedAt: null, + createdAt, + }), + getOrganizationBySlug: async (slug: string) => ({ + id: "org_by_slug", + name: `Org ${slug}`, + slug, + backfilledAt: createdAt, + deletedAt: createdAt, + workosUpdatedAt: null, + createdAt, + }), + markOrganizationDeleted: async () => null, + deleteOrganizationCascade: async () => {}, + }), + ), + }); + const exit = yield* Effect.exit( + resolveBearerAuth(bearer("valid_org_key")).pipe( + Effect.provide( + Layer.mergeAll( + stubApiKeys, + stubWorkOS, + deletedOrgUsers, + stubDirectory, + stubMirror, + stubReadiness, + ), + ), + ), + ); + expect(Exit.isFailure(exit)).toBe(true); + expect( + Exit.isFailure(exit) ? Cause.squash(exit.cause) : null, + "the key outlives the org until the purge; a marked org refuses it", + ).toBeInstanceOf(NoOrganization); + }), + ); + it.effect("user keys still resolve to a bound member principal", () => Effect.gen(function* () { const auth = yield* resolveBearerAuth(bearer("valid_user_key")).pipe(Effect.provide(layers)); @@ -173,10 +279,29 @@ describe("org-level API keys", () => { it.effect("do not trigger a user membership check", () => Effect.gen(function* () { - // `authorizeOrganization` checks a USER's live membership; there is no - // user here. The WorkOS stub dies on any call other than the user path, - // so a clean resolution proves the org branch never took it. - const auth = yield* resolveBearerAuth(bearer("valid_org_key")).pipe(Effect.provide(layers)); + // `authorizeOrganization` checks a USER's membership; there is no user + // here. A directory whose `membership` dies proves the org branch never + // asked. + const noMembershipReads = Layer.succeed(MemberDirectory)({ + membership: () => Effect.die("an org key must not trigger a membership check"), + membershipById: () => Effect.die("an org key must not trigger a membership check"), + membershipsOf: () => Effect.die("an org key must not trigger a membership check"), + members: () => Effect.die("an org key must not trigger a membership check"), + membersById: () => Effect.die("an org key must not trigger a membership check"), + findByEmail: () => Effect.die("an org key must not trigger a membership check"), + }); + const auth = yield* resolveBearerAuth(bearer("valid_org_key")).pipe( + Effect.provide( + Layer.mergeAll( + stubApiKeys, + stubWorkOS, + stubUsers, + noMembershipReads, + stubMirror, + stubReadiness, + ), + ), + ); expect(isPlatformAuth(auth)).toBe(true); }), diff --git a/apps/cloud/src/auth/org-selector-auth.node.test.ts b/apps/cloud/src/auth/org-selector-auth.node.test.ts index 050b483a0e..9cdbd0950e 100644 --- a/apps/cloud/src/auth/org-selector-auth.node.test.ts +++ b/apps/cloud/src/auth/org-selector-auth.node.test.ts @@ -1,18 +1,24 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer } from "effect"; +import type * as Tracer from "effect/Tracer"; + +import { MemberDirectory, type DirectoryMember } from "@executor-js/api/server"; import { ApiKeyService } from "./api-keys"; import { UserStoreService } from "./context"; +import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness"; +import { AUTHORIZE_ORGANIZATION_SPAN } from "./organization"; import { resolveSessionPrincipal } from "./workos-auth-provider"; import { WorkOSClient, type WorkOSClientService } from "./workos"; +import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror"; // The org a console request resolves to is the URL's org (sent in the // `x-executor-organization` selector header) — NEVER the session's stored org. // The sealed cookie's org is a browser-global pinned to whichever org WorkOS // last touched, so a fallback to it silently scopes a multi-org user's request -// to the wrong org; a header-less request fails closed instead. Live -// membership is re-checked either way. This is what makes two browser tabs on -// different orgs independent. +// to the wrong org; a header-less request fails closed instead. Membership is +// re-checked against the local mirror either way. This is what makes two +// browser tabs on different orgs independent. const createdAt = new Date("2026-01-01T00:00:00.000Z"); @@ -30,10 +36,53 @@ const bareAccount = (id: string) => ({ }); // user_session belongs to BOTH orgs; the URL selects which one a request hits. +// Their membership in PENDING_ORG is only pending — an invite, not access. const MEMBER = "user_session"; const SESSION_ORG = "org_session"; const URL_ORG = "org_url"; +const PENDING_ORG = "org_pending"; const URL_SLUG = "acme"; +const PENDING_SLUG = "pending-acme"; + +const mirrored = ( + organizationId: string, + overrides: Partial = {}, +): DirectoryMember => ({ + accountId: MEMBER, + membershipId: `om_${MEMBER}_${organizationId}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active", + lastActiveAt: null, + ...overrides, +}); + +// The mirror as the directory reads it: MEMBER is active in both real orgs, +// an admin of URL_ORG, and merely invited to PENDING_ORG. +const memberships = new Map([ + [SESSION_ORG, mirrored(SESSION_ORG)], + [URL_ORG, mirrored(URL_ORG, { role: "admin" })], + [PENDING_ORG, mirrored(PENDING_ORG, { status: "pending" })], +]); + +// The mirror is READY in these tests (backfill complete, reconciler caught +// up), so membership is read from the stubbed directory, never from WorkOS. +const stubReadiness = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: (accountId, organizationId) => + Effect.succeed(accountId === MEMBER ? (memberships.get(organizationId) ?? null) : null), + membershipById: () => Effect.die("session resolution does not look up by membership id"), + membershipsOf: () => Effect.die("session resolution reads one membership, not the list"), + members: () => Effect.die("session resolution does not list members"), + membersById: () => Effect.die("session resolution does not batch members"), + findByEmail: () => Effect.die("session resolution does not resolve emails"), +}); const stubApiKeys = Layer.succeed(ApiKeyService)({ // No Authorization header in these tests → the api-key path returns null and @@ -59,18 +108,8 @@ const stubWorkOS = Layer.succeed( organizationId: SESSION_ORG, }); } - if (prop === "listUserMemberships") { - return (userId: string) => - Effect.succeed({ - data: - userId === MEMBER - ? [ - { userId, organizationId: SESSION_ORG, status: "active" }, - { userId, organizationId: URL_ORG, status: "active" }, - ] - : [], - }); - } + // Membership is read from the mirror, never from WorkOS: any WorkOS + // call past session authentication fails the test. return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); }, }), @@ -86,7 +125,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: org.id, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -95,32 +134,165 @@ const stubUsers = Layer.succeed(UserStoreService)({ id, name: `Org ${id}`, slug: id, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, }), - // The URL slug maps to URL_ORG (the member's other org); any other slug - // maps to an org the caller is NOT a member of, so membership rejects it. + // The URL slug maps to URL_ORG (the member's other org), the pending + // slug to the org they are only invited to; any other slug maps to an + // org the caller is NOT a member of, so membership rejects it. getOrganizationBySlug: async (slug: string) => ({ - id: slug === URL_SLUG ? URL_ORG : "org_outsider", + id: slug === URL_SLUG ? URL_ORG : slug === PENDING_SLUG ? PENDING_ORG : "org_outsider", name: `Org ${slug}`, slug, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, }), + markOrganizationDeleted: async () => null, deleteOrganizationCascade: async () => {}, }), ), }); -const run = (headers: Record) => +// Authorization scans an organization the backfill never covered before it +// reads the mirror (`auth/organization.ts`); every org row above is marked +// backfilled, so the scan is never reached and the mirror is never written. +const stubMirror = Layer.succeed( + WorkOsMirror, + new Proxy({} as WorkOsMirrorShape, { + get: (_target, prop) => () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`), + }), +); + +const run = ( + headers: Record, + readiness: Layer.Layer = stubReadiness, +) => resolveSessionPrincipal(new Request("https://executor.test/api/tools", { headers })).pipe( - Effect.provide(Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers)), + Effect.provide( + Layer.mergeAll(stubApiKeys, stubWorkOS, stubUsers, stubDirectory, stubMirror, readiness), + ), + ); + +// The mirror before the cutover has landed: the backfill has not covered +// every organization, or the reconciler has not drained recently. +const unreadyMirror = (state: MirrorReadinessState) => + Layer.succeed(MirrorReadiness)({ state: () => Effect.succeed(state) }); + +/** + * A WorkOS that answers the pre-cutover membership list for MEMBER — active + * in SESSION_ORG only, as a member — and records the calls, so the fallback + * is assertable: with the mirror unready the list is read from WorkOS and + * the directory (which says MEMBER is active in URL_ORG too) is never asked. + */ +const workosMemberships = (calls: string[]) => + Layer.succeed( + WorkOSClient, + new Proxy({} as WorkOSClientService, { + get: (_t, prop) => { + if (prop === "authenticateRequest") { + return () => + Effect.succeed({ + userId: MEMBER, + email: "u@e2e.test", + organizationId: SESSION_ORG, + }); + } + if (prop === "listUserMemberships") { + return (userId: string) => + Effect.sync(() => { + calls.push(`listUserMemberships:${userId}`); + return { + object: "list" as const, + data: [ + { + id: `om_${MEMBER}_${SESSION_ORG}`, + userId: MEMBER, + organizationId: SESSION_ORG, + status: "active", + role: { slug: "member" }, + }, + ] as never[], + listMetadata: { before: null, after: null }, + }; + }); + } + return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); + }, + }), ); +const unreadDirectory = Layer.succeed(MemberDirectory)({ + membership: () => Effect.die("an unready mirror must not be asked for membership"), + membershipById: () => Effect.die("an unready mirror must not be asked for membership"), + membershipsOf: () => Effect.die("an unready mirror must not be asked for membership"), + members: () => Effect.die("an unready mirror must not be asked for membership"), + membersById: () => Effect.die("an unready mirror must not be asked for membership"), + findByEmail: () => Effect.die("an unready mirror must not be asked for membership"), +}); + +/** + * A tracer that keeps every span's attributes, so the readiness decision the + * authorization stamps on its span (`mirror.ready`, `mirror.readiness`) is + * assertable: that attribute is what production counts the fallback by. + */ +const makeRecordingTracer = () => { + const spans: { readonly name: string; readonly attributes: Map }[] = []; + const tracer: Tracer.Tracer = { + span: (options) => { + const attributes = new Map(); + spans.push({ name: options.name, attributes }); + let status: Tracer.SpanStatus = { _tag: "Started", startTime: options.startTime }; + return { + _tag: "Span", + name: options.name, + spanId: `span-${spans.length}`, + traceId: "trace-1", + parent: options.parent, + annotations: options.annotations, + get status() { + return status; + }, + attributes, + links: options.links, + sampled: options.sampled, + kind: options.kind, + end: (endTime, exit) => { + status = { _tag: "Ended", startTime: options.startTime, endTime, exit }; + }, + attribute: (key, value) => { + attributes.set(key, value); + }, + event: () => undefined, + addLinks: () => undefined, + }; + }, + }; + const attributesOf = (name: string) => spans.find((span) => span.name === name)?.attributes; + return { tracer, attributesOf }; +}; + +const runAgainstWorkOs = (headers: Record, state: MirrorReadinessState) => { + const calls: string[] = []; + return resolveSessionPrincipal(new Request("https://executor.test/api/tools", { headers })) + .pipe( + Effect.provide( + Layer.mergeAll( + stubApiKeys, + workosMemberships(calls), + stubUsers, + unreadDirectory, + stubMirror, + unreadyMirror(state), + ), + ), + ) + .pipe(Effect.map((principal) => ({ principal, calls }))); +}; + describe("resolveSessionPrincipal · URL org selector", () => { it.effect("fails closed when no selector header is sent", () => Effect.gen(function* () { @@ -141,6 +313,23 @@ describe("resolveSessionPrincipal · URL org selector", () => { "x-executor-organization": URL_SLUG, }); expect(principal.organizationId, "the slug header wins over the session org").toBe(URL_ORG); + expect(principal.orgRole, "the mirrored role binds the executor's write authority").toBe( + "admin", + ); + }), + ); + + it.effect("rejects a selector for an org where the membership is only pending", () => + Effect.gen(function* () { + // An invite is mirrored as a pending membership; it grants no access + // until accepted. + const error = yield* Effect.flip( + run({ + cookie: "wos-session=x", + "x-executor-organization": PENDING_SLUG, + }), + ); + expect(error).toMatchObject({ _tag: "NoOrganization" }); }), ); @@ -168,3 +357,80 @@ describe("resolveSessionPrincipal · URL org selector", () => { }), ); }); + +// The mirror authorizes only while it is READY (`mirror-readiness.ts`): +// the backfill has written every organization and the reconciler has drained +// the stream within its lag budget. Until then the membership check reads +// WorkOS, as it did before the cutover — so a member the backfill has not +// written yet is not locked out, and a stale mirror row cannot grant access +// WorkOS has revoked. +describe("resolveSessionPrincipal · mirror readiness", () => { + const unready: readonly [string, MirrorReadinessState][] = [ + ["the backfill has not completed", MirrorReadinessState.BackfillPending()], + ["the reconciler has never drained", MirrorReadinessState.ReconcilerStale({ drainedAt: null })], + [ + "the reconciler's last drain is older than the budget", + MirrorReadinessState.ReconcilerStale({ drainedAt: createdAt }), + ], + ]; + + for (const [why, state] of unready) { + it.effect(`reads membership from WorkOS, not the mirror, while ${why}`, () => + Effect.gen(function* () { + // WorkOS says MEMBER is active in SESSION_ORG: authorized there... + const granted = yield* runAgainstWorkOs( + { cookie: "wos-session=x", "x-executor-organization": SESSION_ORG }, + state, + ); + expect(granted.principal.organizationId).toBe(SESSION_ORG); + expect(granted.principal.orgRole, "the role comes from WorkOS's list too").toBe("member"); + expect(granted.calls).toEqual([`listUserMemberships:${MEMBER}`]); + + // ...and NOT in URL_ORG, even though the (unready) mirror holds an + // active admin membership there: the mirror's word is not taken. + const refused = yield* Effect.flip( + runAgainstWorkOs({ cookie: "wos-session=x", "x-executor-organization": URL_SLUG }, state), + ); + expect(refused).toMatchObject({ _tag: "NoOrganization" }); + }), + ); + } + + it.effect("reads the mirror once it is ready, without any WorkOS membership call", () => + Effect.gen(function* () { + // `stubWorkOS` dies on any call past session authentication, so a + // resolved principal here proves the list was never requested. + const principal = yield* run( + { cookie: "wos-session=x", "x-executor-organization": URL_SLUG }, + stubReadiness, + ); + expect(principal.organizationId).toBe(URL_ORG); + expect(principal.orgRole).toBe("admin"); + }), + ); + + it.effect("stamps which source answered on the authorization span", () => + Effect.gen(function* () { + // The fallback is counted in production by this attribute, not by + // grepping the warning it logs beside it — so both branches must set it. + const ready = makeRecordingTracer(); + yield* run({ cookie: "wos-session=x", "x-executor-organization": URL_SLUG }).pipe( + Effect.withTracer(ready.tracer), + ); + const readyAttrs = ready.attributesOf(AUTHORIZE_ORGANIZATION_SPAN); + expect(readyAttrs?.get("mirror.ready"), "the mirror answered").toBe(true); + expect(readyAttrs?.get("mirror.readiness")).toBe("ready"); + + const stale = makeRecordingTracer(); + yield* runAgainstWorkOs( + { cookie: "wos-session=x", "x-executor-organization": SESSION_ORG }, + MirrorReadinessState.ReconcilerStale({ drainedAt: null }), + ).pipe(Effect.withTracer(stale.tracer)); + const staleAttrs = stale.attributesOf(AUTHORIZE_ORGANIZATION_SPAN); + expect(staleAttrs?.get("mirror.ready"), "WorkOS answered").toBe(false); + expect(String(staleAttrs?.get("mirror.readiness")), "and the span says why").toContain( + "reconciler", + ); + }), + ); +}); diff --git a/apps/cloud/src/auth/organization.ts b/apps/cloud/src/auth/organization.ts index 68d70fad28..f1d558932b 100644 --- a/apps/cloud/src/auth/organization.ts +++ b/apps/cloud/src/auth/organization.ts @@ -3,7 +3,8 @@ // // One module for the cloud org auth-resolution path: // - `resolveOrganization` — local mirror with lazy WorkOS fallback. -// - `authorizeOrganization` — live membership check, returns the resolved org. +// - `authorizeOrganization` — membership check against the local membership +// mirror, returns the resolved org. // // Deliberately billing-FREE: this module is reached by the MCP session DO bundle // (via `mcp/auth.ts`), which must not transitively import any billing config @@ -11,10 +12,14 @@ // which DO depend on the Autumn plan config — live in `extensions/billing/plans.ts`. // --------------------------------------------------------------------------- -import { Effect } from "effect"; +import { Clock, Effect } from "effect"; +import { MemberDirectory } from "@executor-js/api/server"; import { EXECUTOR_ORG_SELECTOR_HEADER } from "@executor-js/sdk/shared"; import { UserStoreService } from "./context"; +import { ensureOrganizationBackfilled } from "./mirror-feeders"; +import { MirrorReadiness, MirrorReadinessState, describeMirrorReadiness } from "./mirror-readiness"; +import type { Organization } from "./user-store"; import { WorkOSClient } from "./workos"; // --------------------------------------------------------------------------- @@ -53,55 +58,245 @@ export const resolveOrganization = (organizationId: string) => }); // --------------------------------------------------------------------------- -// Authorization — live membership check against WorkOS. +// Deletion mark — the local step that revokes an organization. +// --------------------------------------------------------------------------- +// +// Membership is authorized from the local mirror (below), so deleting the +// WorkOS organization revokes nothing here by itself: the local membership +// rows keep authorizing sessions until the local purge removes them, and a +// purge that fails leaves them live. This mark is what revokes access, and it +// is the FIRST step of cloud's deletion flow (`auth/handlers.ts` +// deleteOrganization) — before the WorkOS delete and the purge, both of which +// can fail — and what the `organization.deleted` event applies +// (`workos-events-sync.ts`) when the org was deleted in the WorkOS dashboard +// instead. Membership rows are left as they are; the mark alone refuses them. +// Idempotent: a retry keeps the first mark. An org the mirror does not hold +// is not marked (nothing to revoke), and `false` says so. + +export const markOrganizationDeleted = (organizationId: string) => + Effect.gen(function* () { + const users = yield* UserStoreService; + const at = new Date(yield* Clock.currentTimeMillis); + const marked = yield* users.use("markOrganizationDeleted", (s) => + s.markOrganizationDeleted(organizationId, at), + ); + return marked !== null; + }); + +// --------------------------------------------------------------------------- +// Authorization — membership check against the local membership mirror. // --------------------------------------------------------------------------- // // The sealed session cookie carries an organizationId that WorkOS signed at // login / refresh time. WorkOS does NOT invalidate existing sessions when a // membership is revoked, and `session.authenticate()` validates the JWT -// locally without hitting the API — so a removed user keeps full access -// until their access token naturally expires (~10 min). +// locally without hitting the API — so a removed user would keep full access +// until their access token naturally expired (~10 min) if the session were +// trusted on its own. +// +// To close that gap, membership is verified on every protected request — but +// against the LOCAL mirror of WorkOS memberships (`memberships` join +// `accounts`, read through the shared `MemberDirectory`), never against WorkOS +// itself. This used to be one `listUserMemberships` call per request (2026-07: +// deliberately NOT cached, because a positive TTL cache is exactly what would +// re-open the revocation gap). The mirror is not a cache with a TTL; it is a +// replica whose freshness is defined by its feeders: +// - login (`auth/handlers.ts` callback): the user and every membership WorkOS +// lists for them, from the list the callback already fetches; +// - write-through: every membership change Executor makes (create org, +// invite, accept, remove, change role) lands in the mirror in the same +// request, so a revocation through Executor is denied on the NEXT request; +// - the WorkOS Events API reconciler (`workos-events-sync.ts`, every minute +// by cron plus a signed webhook poke): changes made in the WorkOS +// dashboard land within seconds. +// The membership row must be `active`: a pending invitee is not a member, and a +// deactivated member keeps their row but not their access. And the +// organization must not be marked deleted (`organizations.deleted_at`): cloud's +// deletion flow (`auth/handlers.ts` deleteOrganization) sets that mark FIRST, +// before the WorkOS delete and the local purge, so an org whose deletion did +// not finish refuses every session at once — its membership rows are still +// there, live, until the purge removes them, and must not authorize anyone. // -// To close that gap we verify membership live on every protected request. -// `listUserMemberships` is one WorkOS call per request. +// The mirror is trusted only while it is READY (`mirror-readiness.ts`): the +// one-off backfill has written every organization, and the events reconciler +// has drained the stream within its lag budget. Until both hold, membership is +// read from WorkOS (`listUserMemberships`, one call per request) exactly as +// before the cutover — a member the backfill has not written yet must not be +// locked out, and a member revoked in the dashboard while the reconciler was +// down must not be let in on a stale row. The readiness row is one indexed +// point read on the same socket; the deploy gate +// (`scripts/ensure-workos-mirror-ready.ts`) applies the same rule before this +// build goes live, so in steady state the fallback is never taken. A +// readiness or mirror read failure fails the request (500), never a silent +// fallback in either direction. The one org the fallback never asks WorkOS +// about is one the mirror holds as DELETED: WorkOS no longer has it (or is +// about to not), so its answer is "no member" for everyone — including the +// admin whose deletion failed part-way and must retry it (below). That +// membership is read from the mirror, whose rows are exactly what the purge +// has not removed yet, ready or not; a refused caller gets null either way. // -// Caching decision (2026-07): we deliberately do NOT add a positive TTL cache -// here. A positive cache is exactly what would re-open the revocation gap this -// live check exists to close — a revoked member would keep access for the cache -// TTL. Negative caching is worse still (a transient WorkOS blip would get -// pinned as "no access"), so it is out too. The rate-limit amplification a -// shared-API-key org can cause under a WorkOS slowdown is mitigated instead by -// the classification fix at the MCP call site (a blip now yields a retryable -// 503, so it no longer condemns sessions or triggers reconnect storms). If per- -// request WorkOS load later proves to be the bottleneck, the right structural -// fix is a local memberships table fed by the WorkOS Events API (authoritative, -// no staleness window), not a TTL cache over this call — tracked as follow-up. +// Readiness is database-wide; completeness is PER ORGANIZATION. An +// organization whose row was minted after the backfill ran — lazily by a +// request (`resolveOrganization`), or by a first login — carries no +// `backfilled_at`, and the mirror holds only the memberships login and +// write-through happened to record for it: a member who has not signed in +// since would be refused on a row that was never written. So the org row is +// read FIRST, and an unmarked live organization is scanned from WorkOS +// (`ensureOrganizationBackfilled`: one membership listing plus one `getUser` +// per member, then the mark) BEFORE its mirror is read — the same on-demand +// scan the seat gates run. One-time per organization: the scan marks the +// row, and this branch is never taken for it again. An organization the +// mirror does not hold at all — one that predates the mirror and that nobody +// has signed in to since (a CLI or MCP token names it, and the JWT path has +// no login feeder), or one created in the WorkOS dashboard — is reachable by +// neither the backfill (which lists the mirror's organizations) nor the +// reconciler (which starts at the replay boundary), so it is resolved on +// demand HERE: WorkOS is asked for the caller's own membership in it first +// (`getUserOrgMembership`, a read scoped to this caller — never a listing +// of the org), and only a member's answer mints the row +// (`resolveOrganization`) and scans it as above. A non-member mints nothing: +// a signed-in caller cannot create the row of an arbitrary WorkOS +// organization by naming its id. An organization marked deleted is never +// scanned: WorkOS no longer has it, and its rows are the purge's to remove, +// not a listing's to refresh. // -// Returns the resolved organization (via resolveOrganization) if the user -// currently holds an *active* membership in it, otherwise null. Callers -// should treat null as "no access" and route accordingly (onboarding page / -// 403). +// Returns the resolved organization if the user currently holds an *active* +// membership in it, otherwise null. Callers should treat null as "no access" +// and route accordingly (onboarding page / 403). +// +// The ONE caller that may see a marked org is the deletion flow itself +// (`deleted: "allow"`): an admin whose deletion failed after the mark must be +// able to send it again to finish the purge, and their membership row is +// still there to authorize exactly that. + +export interface AuthorizeOrganizationOptions { + /** Whether an organization marked deleted resolves (`"allow"`) or is refused (default). */ + readonly deleted?: "refuse" | "allow"; +} + +/** The caller's active membership in the org, however it was read: only the role matters past this point. */ +interface ActiveMembership { + readonly role: string; +} + +// The mirror read: the caller's row, active or nothing. +const activeMembershipFromMirror = (userId: string, organizationId: string) => + Effect.gen(function* () { + const directory = yield* MemberDirectory; + const membership = yield* directory.membership(userId, organizationId); + if (!membership || membership.status !== "active") return null; + const active: ActiveMembership = { role: membership.role }; + return active; + }); -export const authorizeOrganization = (userId: string, organizationId: string) => +// The pre-cutover read, kept for the window in which the mirror is not yet +// ready: WorkOS's own membership list for the user, one call per request. +const activeMembershipFromWorkOs = (userId: string, organizationId: string) => Effect.gen(function* () { const workos = yield* WorkOSClient; const memberships = yield* workos.listUserMemberships(userId); - const active = memberships.data.find( - (m: { readonly organizationId: string; readonly status: string }) => - m.organizationId === organizationId && m.status === "active", + const membership = memberships.data.find( + (m) => m.organizationId === organizationId && m.status === "active", ); - if (!active) return null; - - const org = yield* resolveOrganization(organizationId); - // The membership row already names the caller's role — surface it - // normalized so identity resolution can bind the executor's workspace - // write permission without a second WorkOS call. WorkOS issues - // `admin` / `member`; anything unrecognized stays a plain member. - const roleSlug = (active as { readonly role?: { readonly slug?: string } }).role?.slug; - const memberRole: "admin" | "member" = roleSlug === "admin" ? "admin" : "member"; - return { ...org, memberRole }; + if (!membership) return null; + const active: ActiveMembership = { role: membership.role.slug }; + return active; }); +// The authorized organization, or null for one marked deleted (unless the +// caller is the deletion flow). The membership already names the caller's +// role — surfaced normalized so identity resolution can bind the executor's +// workspace write permission without a second read. WorkOS issues `admin` / +// `member`; anything unrecognized stays a plain member. +const authorized = ( + org: Organization, + membership: ActiveMembership, + options: AuthorizeOrganizationOptions, +) => { + if (org.deletedAt !== null && options.deleted !== "allow") return null; + const memberRole: "admin" | "member" = membership.role === "admin" ? "admin" : "member"; + return { ...org, memberRole }; +}; + +// The organization row for a caller, minted from WorkOS when the mirror +// does not hold it — only for a caller WorkOS confirms as its member (see +// above). `null` when the mirror has no row and WorkOS lists no membership. +const heldOrResolvedForMember = (userId: string, organizationId: string) => + Effect.gen(function* () { + const users = yield* UserStoreService; + const held = yield* users.use("getOrganization", (s) => s.getOrganization(organizationId)); + if (held) return held; + const workos = yield* WorkOSClient; + const membership = yield* workos.getUserOrgMembership(organizationId, userId); + if (!membership) return null; + yield* Effect.logInfo( + "authorizeOrganization: organization not mirrored; resolving it from WorkOS for its member", + { organizationId }, + ); + return yield* resolveOrganization(organizationId); + }); + +/** + * The span every membership authorization runs under, with the readiness + * decision stamped on it so the fallback can be counted rather than grepped: + * `mirror.ready` (boolean, which source answered) and `mirror.readiness` (the + * state's description, why). Query Axiom for `name == "auth.authorize_organization"` + * and `mirror.ready == false` to see how many requests are on the WorkOS + * fallback and for which reason. + */ +export const AUTHORIZE_ORGANIZATION_SPAN = "auth.authorize_organization"; + +export const authorizeOrganization = ( + userId: string, + organizationId: string, + options: AuthorizeOrganizationOptions = {}, +) => + Effect.gen(function* () { + const readiness = yield* MirrorReadiness; + const state = yield* readiness.state(); + const ready = MirrorReadinessState.$is("Ready")(state); + yield* Effect.annotateCurrentSpan({ + "mirror.ready": ready, + "mirror.readiness": describeMirrorReadiness(state), + }); + if (!ready) { + yield* Effect.logWarning( + "authorizeOrganization: membership mirror not ready; membership read from WorkOS", + { readiness: describeMirrorReadiness(state) }, + ); + // A marked organization is the mirror's to answer for (see above): + // WorkOS lists no member of it, and the deletion retry must still + // get in. + const users = yield* UserStoreService; + const held = yield* users.use("getOrganization", (s) => s.getOrganization(organizationId)); + if (held?.deletedAt != null) { + if (options.deleted !== "allow") return null; + const membership = yield* activeMembershipFromMirror(userId, organizationId); + if (!membership) return null; + return authorized(held, membership, options); + } + const membership = yield* activeMembershipFromWorkOs(userId, organizationId); + if (!membership) return null; + // The row read above is reused: `resolveOrganization` would read it a + // second time, and a request reads the organization row ONCE (the MCP + // session DO relies on that — see e2e `mcp-session-cold-init`). + const org = held ?? (yield* resolveOrganization(organizationId)); + return authorized(org, membership, options); + } + + const org = yield* heldOrResolvedForMember(userId, organizationId); + if (!org) return null; + // An unmarked live organization is scanned before its mirror is read + // (see above). The row returned below still shows the mark as it was + // read; nothing past this point reads it. + if (org.deletedAt === null && org.backfilledAt === null) { + yield* ensureOrganizationBackfilled(organizationId); + } + const membership = yield* activeMembershipFromMirror(userId, organizationId); + if (!membership) return null; + return authorized(org, membership, options); + }).pipe(Effect.withSpan(AUTHORIZE_ORGANIZATION_SPAN)); + // --------------------------------------------------------------------------- // Org SELECTOR — the URL is the scope authority, not the session. // --------------------------------------------------------------------------- @@ -111,8 +306,8 @@ export const authorizeOrganization = (userId: string, organizationId: string) => // its own `x-executor-mcp-organization`). The selector is a slug (`acme`, the // readable URL form) or a WorkOS id (`org_…`, the legacy/token form). It is a // SELECTOR, not a trust boundary: `authorizeOrganizationSelector` re-checks -// live membership, so the worst a forged header does is name an org the caller -// already belongs to. +// membership against the mirror, so the worst a forged header does is name an +// org the caller already belongs to. // // Why a header and not the session's `org_id`: a browser shares ONE cookie jar // across tabs, so a single session-pinned org makes "active org" a @@ -130,15 +325,19 @@ export const orgSelectorFromRequest = (request: Request): string | null => * Resolve an org SELECTOR (URL slug or `org_…` id) to the organization the * caller actively belongs to, or `null`. A slug resolves through the local * mirror to its id first; ids pass straight through. Either way membership is - * verified live via {@link authorizeOrganization}. + * verified against the mirror via {@link authorizeOrganization}. */ -export const authorizeOrganizationSelector = (userId: string, selector: string) => +export const authorizeOrganizationSelector = ( + userId: string, + selector: string, + options: AuthorizeOrganizationOptions = {}, +) => Effect.gen(function* () { if (selector.startsWith("org_")) { - return yield* authorizeOrganization(userId, selector); + return yield* authorizeOrganization(userId, selector, options); } const users = yield* UserStoreService; const org = yield* users.use("getOrganizationBySlug", (s) => s.getOrganizationBySlug(selector)); if (!org) return null; - return yield* authorizeOrganization(userId, org.id); + return yield* authorizeOrganization(userId, org.id, options); }); diff --git a/apps/cloud/src/auth/user-store.ts b/apps/cloud/src/auth/user-store.ts index 1dfdb7d022..ec4fd862b8 100644 --- a/apps/cloud/src/auth/user-store.ts +++ b/apps/cloud/src/auth/user-store.ts @@ -7,7 +7,7 @@ // so domain tables can foreign-key against them and so we can resolve org // metadata without an API call on every request. -import { and, eq, isNull, lte, or } from "drizzle-orm"; +import { and, eq, isNull, lte, or, sql } from "drizzle-orm"; import { generateOrgSlug } from "@executor-js/api"; @@ -148,6 +148,24 @@ export const makeUserStore = (db: DrizzleDb) => { return rows[0] ?? null; }, + // Mark an org deleted, refusing every membership authorization against + // it from this moment. The FIRST step of cloud's deletion flow, taken + // before the WorkOS delete and the local purge, so a failure in either + // later step leaves the org unreachable rather than still authorizing + // sessions from its live membership rows. Idempotent: a retry after the + // WorkOS org is already gone keeps the original mark. `null` when the + // org is not mirrored. + markOrganizationDeleted: async (id: string, at: Date): Promise => { + const [marked] = await db + .update(organizations) + .set({ + deletedAt: sql`coalesce(${organizations.deletedAt}, ${at.toISOString()}::timestamptz)`, + }) + .where(eq(organizations.id, id)) + .returning(); + return marked ?? null; + }, + // Permanently delete everything an org owns (tenant data, secrets, its // memberships) in a single transaction, leaving the organization row as // a tombstone marked `deletedAt` (see `purgeOrganizationData` for why). diff --git a/apps/cloud/src/auth/workos-auth-provider.ts b/apps/cloud/src/auth/workos-auth-provider.ts index 6abbffb7cb..25ce48a54c 100644 --- a/apps/cloud/src/auth/workos-auth-provider.ts +++ b/apps/cloud/src/auth/workos-auth-provider.ts @@ -19,13 +19,17 @@ // - session without org header -> NoOrganization 403 no_organization (fail closed) // - session org not authorized -> NoOrganization 403 no_organization // - no auth header -> falls through to the sealed-session path -// The org-resolution infra errors (`UserStoreError` / `WorkOSError`) are -// `Effect.die`d so they surface as 500 defects — the same status the old inline -// resolver produced when those bubbled up. +// The org-resolution infra errors (`UserStoreError` / `WorkOSError` / +// `MemberDirectoryError` / `WorkOsMirrorError`, the last from the mirror +// readiness read) are `Effect.die`d so they surface as 500 defects — the +// same status the old inline resolver produced when those bubbled up. // -// The per-request `UserStoreService` (read by the org-resolution path) stays a -// REQUIREMENT OF THE LAYER, satisfied by the facade's per-request DB combine — -// NOT a function-level requirement (that is what forced a forked tag before). +// The per-request `UserStoreService` + `MemberDirectory` + `MirrorReadiness` +// + `WorkOsMirror` (read by the org-resolution path: the org row, whether +// the mirror may be trusted, the caller's mirrored membership, and the +// on-demand scan of an organization the backfill never covered) stay +// REQUIREMENTS OF THE LAYER, satisfied by the facade's per-request DB combine — +// NOT function-level requirements (that is what forced a forked tag before). // --------------------------------------------------------------------------- import { Effect, Layer } from "effect"; @@ -34,6 +38,7 @@ import type { JWTVerifyGetKey } from "jose"; import { IdentityProvider, + MemberDirectory, NoOrganization, Unauthorized, Unavailable, @@ -41,6 +46,7 @@ import { import type { FailureRenderingStrategy, IdentityFailure, + MemberDirectoryError, PlatformPrincipal, Principal, ResolvedPrincipal, @@ -48,6 +54,8 @@ import type { import { ApiKeyService } from "./api-keys"; import { workosApiJwtBearerConfig } from "./api-jwt-bearer"; +import { MirrorReadiness } from "./mirror-readiness"; +import { WorkOsMirror } from "./workos-mirror"; import { BEARER_PREFIX } from "./bearer"; import { authorizeOrganization, @@ -57,7 +65,7 @@ import { } from "./organization"; import { UserStoreService } from "./context"; import { sealedSessionDisplayName } from "./middleware"; -import type { UserStoreError, WorkOSError } from "./errors"; +import type { UserStoreError, WorkOSError, WorkOsMirrorError } from "./errors"; import { WorkOSClient } from "./workos"; import { verifyWorkosUserManagementToken } from "../mcp/jwt"; @@ -66,7 +74,7 @@ import { verifyWorkosUserManagementToken } from "../mcp/jwt"; * (user_management) access token: the client-scoped SSO JWKS resolver. Issuer * and audience are NOT pinned (the client-scoped JWKS binds the token to this * app; user_management tokens carry no audience and an app-specific issuer) and - * org membership is re-checked live downstream. Passed in as a plain value so + * org membership is re-checked against the mirror downstream. Passed in as a plain value so * this module stays `cloudflare:workers`-free and the node-pool resolver tests * can inject a local JWKS. Production supplies {@link workosApiJwtBearerConfig}. */ @@ -118,7 +126,7 @@ const looksLikeJwt = (token: string): boolean => token.split(".").length === 3; /** * Resolve a WorkOS device-login (user_management) access token into a protected * `Principal`. Verifies the token's signature + expiry against the client-scoped - * SSO JWKS, then live-checks org membership, exactly like the api-key path. The + * SSO JWKS, then checks org membership in the mirror, exactly like the api-key path. The * `org_id` claim must be present (a token with no org context is rejected as * `NoOrganization`). NOTE: this is a different WorkOS token domain than the MCP * `/oauth2` tokens (different keyset, no audience), so it does NOT reuse the MCP @@ -191,7 +199,7 @@ export const isPlatformAuth = (value: BearerAuth): value is PlatformAuth => * path. * * The org branch does NOT call `authorizeOrganization`: that checks a USER's - * live membership, and there is no user here. The key itself is the authority — + * membership, and there is no user here. The key itself is the authority — * WorkOS validated it and reported which org owns it — so the org row is merely * resolved (mirrored on first read) for its name and slug. */ @@ -200,8 +208,14 @@ export const resolveBearerAuth = ( jwt: JwtBearerConfig | null = null, ): Effect.Effect< BearerAuth, - Unauthorized | NoOrganization | Unavailable | UserStoreError | WorkOSError, - WorkOSClient | ApiKeyService | UserStoreService + | Unauthorized + | NoOrganization + | Unavailable + | UserStoreError + | WorkOSError + | WorkOsMirrorError + | MemberDirectoryError, + WorkOSClient | ApiKeyService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror > => Effect.gen(function* () { const authHeader = request.headers.get("authorization"); @@ -229,6 +243,9 @@ export const resolveBearerAuth = ( if (owner.scope === "org") { const org = yield* resolveOrganization(owner.organizationId); + // The key outlives the org until the purge removes it; an org marked + // deleted refuses it as it refuses every member's session. + if (org.deletedAt !== null) return yield* new NoOrganization(NO_ORGANIZATION_IN_API_KEY); return { kind: "platform", organizationId: org.id, @@ -277,8 +294,14 @@ export const resolveApiKeyPrincipal = ( jwt: JwtBearerConfig | null = null, ): Effect.Effect< ResolvedPrincipal | null, - Unauthorized | NoOrganization | Unavailable | UserStoreError | WorkOSError, - WorkOSClient | ApiKeyService | UserStoreService + | Unauthorized + | NoOrganization + | Unavailable + | UserStoreError + | WorkOSError + | WorkOsMirrorError + | MemberDirectoryError, + WorkOSClient | ApiKeyService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror > => Effect.gen(function* () { const auth = yield* resolveBearerAuth(request, jwt); @@ -306,8 +329,9 @@ export const resolveSessionPrincipal = (request: Request) => // browser-global and pinned to whichever org WorkOS last touched, so // falling back to it silently serves ANOTHER org's data to a multi-org // user (the wrong-tenant connection-list bug, 2026-07). A header-less - // session call gets a clear 403 instead. Membership is re-checked live — - // the header is a selector, not a trust boundary (see organization.ts). + // session call gets a clear 403 instead. Membership is re-checked against + // the mirror — the header is a selector, not a trust boundary (see + // organization.ts). // A bare-URL first paint (no org in the path yet) may 403 here; that's // the safe outcome — OrgSlugGate immediately canonicalizes the URL onto // an org slug, the org-keyed atom registry remounts, and everything @@ -340,9 +364,10 @@ export const resolveSessionPrincipal = (request: Request) => * no roles to resolve, so each leaf already carries `roles: []`. Raises the * SHARED identity errors directly (`Unauthorized | NoOrganization | Unavailable`, * each carrying its machine `code` + `message`); the org-resolution infra errors - * (`UserStoreError` / `WorkOSError`) bubble for `workosIdentityLayer` to `die`. - * Keeps `WorkOSClient` / `ApiKeyService` / `UserStoreService` as requirements (the - * org-resolution path reads them) so it stays request-scoped. Re-exported for + * (`UserStoreError` / `WorkOSError` / `MemberDirectoryError`) bubble for + * `workosIdentityLayer` to `die`. Keeps `WorkOSClient` / `ApiKeyService` / + * `UserStoreService` / `MemberDirectory` as requirements (the org-resolution + * path reads them) so it stays request-scoped. Re-exported for * `protected-api-key-auth.node.test.ts`, which asserts the per-path principal + * shared error codes this folded resolver emits. */ @@ -351,8 +376,14 @@ export const resolveProtectedPrincipal = ( jwt: JwtBearerConfig | null = null, ): Effect.Effect< ResolvedPrincipal, - Unauthorized | NoOrganization | Unavailable | UserStoreError | WorkOSError, - WorkOSClient | ApiKeyService | UserStoreService + | Unauthorized + | NoOrganization + | Unavailable + | UserStoreError + | WorkOSError + | WorkOsMirrorError + | MemberDirectoryError, + WorkOSClient | ApiKeyService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror > => Effect.gen(function* () { const bearerPrincipal = yield* resolveApiKeyPrincipal(request, jwt); @@ -362,22 +393,29 @@ export const resolveProtectedPrincipal = ( /** * Cloud's NEUTRAL `IdentityProvider` Layer. Closes over the long-lived - * `WorkOSClient` + `ApiKeyService`; the request-scoped `UserStoreService` stays a - * REQUIREMENT OF THE LAYER, satisfied per request by the facade's DB combine. - * `authenticate` matches the neutral shape exactly (`Effect`): rejected credentials already - * carry the shared errors; the org-resolution infra errors (`UserStoreError` / - * `WorkOSError`) are `Effect.die`d so they surface as 500 defects, never on the - * error channel. + * `WorkOSClient` + `ApiKeyService`; the request-scoped `UserStoreService` + + * `MemberDirectory` stay REQUIREMENTS OF THE LAYER, satisfied per request by the + * facade's DB combine. `authenticate` matches the neutral shape exactly + * (`Effect`): rejected + * credentials already carry the shared errors; the org-resolution infra errors + * (`UserStoreError` / `WorkOSError` / `MemberDirectoryError`) are `Effect.die`d + * so they surface as 500 defects, never on the error channel. */ export const workosIdentityLayer: Layer.Layer< IdentityProvider, never, - WorkOSClient | ApiKeyService | UserStoreService + WorkOSClient | ApiKeyService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror > = Layer.effect( IdentityProvider, Effect.gen(function* () { - const context = yield* Effect.context(); + const context = yield* Effect.context< + | WorkOSClient + | ApiKeyService + | UserStoreService + | MemberDirectory + | MirrorReadiness + | WorkOsMirror + >(); return IdentityProvider.of({ authenticate: (request) => resolveProtectedPrincipal(request, workosApiJwtBearerConfig).pipe( @@ -390,6 +428,10 @@ export const workosIdentityLayer: Layer.Layer< UserStoreError: (error) => Effect.die(error), // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: org-resolution infra failure -> 500 defect, matches prior inline-resolver behavior WorkOSError: (error) => Effect.die(error), + // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: membership-mirror read failure -> 500 defect, same class as the store failure above + MemberDirectoryError: (error) => Effect.die(error), + // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: mirror-readiness read failure -> 500 defect, same class as the store failure above + WorkOsMirrorError: (error) => Effect.die(error), }), Effect.provide(context), ), diff --git a/apps/cloud/src/auth/workos-callback-state.node.test.ts b/apps/cloud/src/auth/workos-callback-state.node.test.ts index a80d4a710b..d7e56cd672 100644 --- a/apps/cloud/src/auth/workos-callback-state.node.test.ts +++ b/apps/cloud/src/auth/workos-callback-state.node.test.ts @@ -102,6 +102,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ workosUpdatedAt: null, createdAt: new Date(), }), + markOrganizationDeleted: async () => null, deleteOrganizationCascade: async () => {}, }), ), @@ -130,6 +131,8 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ const stubDirectory = Layer.succeed(MemberDirectory)({ membership: () => Effect.die("the callback does not look up one membership"), + membershipById: () => Effect.die("the callback does not look up by membership id"), + membershipsOf: () => Effect.die("the callback reads the WorkOS list, not the mirror's"), membersById: () => Effect.die("the callback does not batch members"), findByEmail: () => Effect.die("the callback does not resolve emails"), members: (organizationId) => diff --git a/apps/cloud/src/auth/workos-events-sync.node.test.ts b/apps/cloud/src/auth/workos-events-sync.node.test.ts index d0a465a09d..6e53189638 100644 --- a/apps/cloud/src/auth/workos-events-sync.node.test.ts +++ b/apps/cloud/src/auth/workos-events-sync.node.test.ts @@ -12,7 +12,8 @@ // deleted, membership created/updated/deleted, organization renamed // - an older event never regresses a newer row (`stale`) — an older // organization rename included — a replayed delete is `absent`, and -// `organization.deleted` MARKS the org deleted without purging anything; +// `organization.deleted` MARKS the org deleted (refusing every membership +// authorization) without purging anything; // replayed, or after cloud's own flow marked it first, it is `absent` // - `organization.deleted` for an org the mirror has never seen MINTS a // tombstone row, so a login that fetched a membership of it before the @@ -61,6 +62,8 @@ import { UserStoreService } from "./context"; import { WorkOSError } from "./errors"; import { cloudMemberDirectoryLayer } from "./member-directory"; import { mirrorSignIn } from "./mirror-feeders"; +import { MirrorReadiness, MirrorReadinessState } from "./mirror-readiness"; +import { authorizeOrganization } from "./organization"; import { WorkOSClient, type WorkOSClientService, type WorkOSListEventsOptions } from "./workos"; import { planEvent, @@ -199,13 +202,26 @@ const profiles = (reads: string[] = []): Partial => ({ }); const DbLive = DbService.Live; +// The mirror is READY here (the authorization checks below read the mirror, +// not WorkOS); the readiness rule is pinned in workos-mirror.node.test.ts. +const readyMirror = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + const MirrorServices = Layer.mergeAll( WorkOsMirror.Live, UserStoreService.Live, cloudMemberDirectoryLayer, + readyMirror, ).pipe(Layer.provideMerge(DbLive)); -type Services = WorkOsMirror | UserStoreService | MemberDirectory | DbService | WorkOSClient; +type Services = + | WorkOsMirror + | UserStoreService + | MemberDirectory + | MirrorReadiness + | DbService + | WorkOSClient; const run = ( body: Effect.Effect, @@ -754,6 +770,15 @@ describe("applyEvent", () => { const result = await run( Effect.gen(function* () { const seeded = yield* seedOrganization(org); + // Marked as scanned (an empty listing at T1), as the one-off backfill + // leaves every org: authorization scans an unmarked org from WorkOS + // first, and no WorkOS read is served here. + const mirror = yield* WorkOsMirror; + yield* mirror.applyOrganizationScan({ + organizationId: org, + listedAt: new Date(T1), + members: [], + }); yield* applyEvent( membershipEvent("organization_membership.created", workosMembership(userId, org)), ); @@ -767,6 +792,7 @@ describe("applyEvent", () => { organizationEvent("organization.updated", workosOrganization(org, "Older Name", T1)), ); const afterOlderRename = yield* readOrganization(org); + const authorizedBefore = yield* authorizeOrganization(userId, org); const deleted = yield* applyEvent( organizationEvent( "organization.deleted", @@ -777,6 +803,7 @@ describe("applyEvent", () => { ); const orgAfterDelete = yield* readOrganization(org); const membershipAfterDelete = yield* readMembership(userId, org); + const authorizedAfter = yield* authorizeOrganization(userId, org); const deletedAgain = yield* applyEvent( organizationEvent( "organization.deleted", @@ -795,9 +822,11 @@ describe("applyEvent", () => { afterRename, olderRename, afterOlderRename, + authorizedBefore, deleted, orgAfterDelete, membershipAfterDelete, + authorizedAfter, deletedAgain, renamedAfterDelete, orgAfterReplay, @@ -810,10 +839,12 @@ describe("applyEvent", () => { expect(result.afterRename?.slug, "the slug is stable across renames").toBe(result.seeded.slug); expect(result.olderRename, "an older rename is refused").toBe("stale"); expect(result.afterOlderRename?.name).toBe("Renamed Org"); + expect(result.authorizedBefore).not.toBeNull(); expect(result.deleted, "organization.deleted marks the org").toBe("applied"); expect(result.orgAfterDelete?.deletedAt, "as of the event").toEqual(new Date(T2)); expect(result.orgAfterDelete?.name, "the row is kept, not purged").toBe("Renamed Org"); expect(result.membershipAfterDelete, "and so is the membership row").not.toBeNull(); + expect(result.authorizedAfter, "but it authorizes nobody any more").toBeNull(); expect(result.deletedAgain, "a replayed deletion changes nothing").toBe("absent"); expect(result.renamedAfterDelete, "a deleted org is never renamed").toBe("absent"); expect(result.orgAfterReplay?.deletedAt, "the first mark stands").toEqual(new Date(T2)); diff --git a/apps/cloud/src/auth/workos-mirror.node.test.ts b/apps/cloud/src/auth/workos-mirror.node.test.ts index f70c208169..8a07eab012 100644 --- a/apps/cloud/src/auth/workos-mirror.node.test.ts +++ b/apps/cloud/src/auth/workos-mirror.node.test.ts @@ -40,12 +40,13 @@ // - the events replay boundary is recorded once and never advanced // - `members` searches email AND name case-insensitively, pages stably // - `findByEmail` ignores the casing WorkOS stored +// - `membershipById` is org-scoped: another org's id resolves to null // - a membership arriving before its user still holds (FK via ensureAccount) // --------------------------------------------------------------------------- import { describe, expect, it } from "@effect/vitest"; import { eq, sql } from "drizzle-orm"; -import { Context, Deferred, Effect, Fiber, Layer, Option } from "effect"; +import { Context, Deferred, Duration, Effect, Fiber, Layer, Option } from "effect"; import { MemberDirectory } from "@executor-js/api/server"; @@ -60,6 +61,13 @@ import { type WorkOsMirrorUser, } from "./workos-mirror"; import { makeWorkOsMirrorStore } from "./workos-mirror-store"; +import { + MIRROR_RECONCILER_LAG_BUDGET, + MirrorReadiness, + MirrorReadinessState, + makeMirrorReadinessLayer, + mirrorReadinessFrom, +} from "./mirror-readiness"; const DbLive = DbService.Live; const Services = Layer.mergeAll( @@ -841,6 +849,62 @@ describe("WorkOsMirror cursor", () => { }); }); +describe("mirror readiness", () => { + const now = T4; + const budget = Duration.toMillis(MIRROR_RECONCILER_LAG_BUDGET); + const within = new Date(now.getTime() - budget); + const tooOld = new Date(now.getTime() - budget - 1); + + it("is ready only when the backfill has completed AND the reconciler drained within the budget", () => { + expect(mirrorReadinessFrom(null, now), "no events row: never backfilled").toEqual( + MirrorReadinessState.BackfillPending(), + ); + expect(mirrorReadinessFrom({ backfillCompletedAt: null, drainedAt: within }, now)).toEqual( + MirrorReadinessState.BackfillPending(), + ); + expect( + mirrorReadinessFrom({ backfillCompletedAt: T1, drainedAt: null }, now), + "backfilled but the reconciler has never drained", + ).toEqual(MirrorReadinessState.ReconcilerStale({ drainedAt: null })); + expect( + mirrorReadinessFrom({ backfillCompletedAt: T1, drainedAt: tooOld }, now), + "a drain older than the budget is stale", + ).toEqual(MirrorReadinessState.ReconcilerStale({ drainedAt: tooOld })); + expect( + mirrorReadinessFrom({ backfillCompletedAt: T1, drainedAt: within }, now), + "a drain exactly at the budget is still ready", + ).toEqual(MirrorReadinessState.Ready()); + expect(mirrorReadinessFrom({ backfillCompletedAt: T1, drainedAt: now }, now)).toEqual( + MirrorReadinessState.Ready(), + ); + }); + + it("reads the live row the backfill and the reconciler write", async () => { + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const readiness = yield* MirrorReadiness; + yield* clearEventsRow; + const noRow = yield* readiness.state(); + yield* mirror.setReplayBoundary(T1); + yield* mirror.markBackfillCompleted(T1); + const backfilledOnly = yield* readiness.state(); + // A drain as of now: what a reconciler run that just read the stream + // to its end records. + const drainedAt = new Date(); + yield* mirror.markDrained(drainedAt); + const ready = yield* readiness.state(); + return { noRow, backfilledOnly, ready }; + }).pipe(Effect.provide(makeMirrorReadinessLayer().pipe(Layer.provide(DbLive)))), + ); + expect(result.noRow).toEqual(MirrorReadinessState.BackfillPending()); + expect(result.backfilledOnly).toEqual( + MirrorReadinessState.ReconcilerStale({ drainedAt: null }), + ); + expect(result.ready).toEqual(MirrorReadinessState.Ready()); + }); +}); + describe("WorkOsMirror backfill sync state", () => { it("records the replay boundary and the backfill completion once each, and the drained mark forward only, without touching the cursor", async () => { const result = await run( @@ -1244,6 +1308,33 @@ describe("cloud MemberDirectory", () => { expect(result.wildcard, "a literal % matches nothing rather than everything").toEqual([]); }); + it("lists one account's memberships across orgs, active + pending by default", async () => { + const result = await run( + Effect.gen(function* () { + const directory = yield* MemberDirectory; + const mirror = yield* WorkOsMirror; + const active = yield* freshOrg(); + const pending = yield* freshOrg(); + const inactive = yield* freshOrg(); + const id = `user_${crypto.randomUUID()}`; + yield* mirror.upsertUser(user(id)); + yield* mirror.upsertMembership(membership(active, id, { role: "admin" })); + yield* mirror.upsertMembership(membership(pending, id, { status: "pending" })); + yield* mirror.upsertMembership(membership(inactive, id, { status: "inactive" })); + const defaults = yield* directory.membershipsOf(id); + const activeOnly = yield* directory.membershipsOf(id, ["active"]); + const nobody = yield* directory.membershipsOf(`user_${crypto.randomUUID()}`); + return { active, pending, inactive, defaults, activeOnly, nobody }; + }), + ); + expect(result.defaults.map((m) => m.organizationId)).toEqual( + [result.active, result.pending].sort(), + ); + expect(result.defaults.find((m) => m.organizationId === result.active)?.role).toBe("admin"); + expect(result.activeOnly.map((m) => m.organizationId)).toEqual([result.active]); + expect(result.nobody).toEqual([]); + }); + it("resolves a normalized email regardless of stored casing, and batches by id", async () => { const result = await run( Effect.gen(function* () { @@ -1264,6 +1355,9 @@ describe("cloud MemberDirectory", () => { ["active", "pending", "inactive"], ); const empty = yield* directory.membersById(org, []); + const byId = yield* directory.membershipById(org, `om_${ids.gone}_${org}`); + const byIdForeign = yield* directory.membershipById(other, `om_${ids.gone}_${org}`); + const byIdUnknown = yield* directory.membershipById(org, "om_unknown"); return { ids, found, @@ -1273,6 +1367,9 @@ describe("cloud MemberDirectory", () => { batch, batchAll, empty, + byId, + byIdForeign, + byIdUnknown, }; }), ); @@ -1285,5 +1382,11 @@ describe("cloud MemberDirectory", () => { ]); expect([...result.batchAll.keys()].sort()).toEqual([result.ids.ada, result.ids.gone].sort()); expect(result.empty.size).toBe(0); + expect(result.byId, "membershipById reports any status").toMatchObject({ + accountId: result.ids.gone, + status: "inactive", + }); + expect(result.byIdForeign, "an id from another org is not this org's").toBeNull(); + expect(result.byIdUnknown).toBeNull(); }); }); diff --git a/apps/cloud/src/auth/workos.ts b/apps/cloud/src/auth/workos.ts index e8bf57c569..918a7e8556 100644 --- a/apps/cloud/src/auth/workos.ts +++ b/apps/cloud/src/auth/workos.ts @@ -685,7 +685,11 @@ const make = Effect.gen(function* () { ), ), - /** Get a user's membership in an organization. */ + /** + * A user's membership in an organization (active or pending), or `null` + * when WorkOS lists none: the user is not a member, or the organization + * is gone. + */ getUserOrgMembership: (organizationId: string, userId: string) => use("userManagement.listOrganizationMemberships", async (wos) => { const response = await wos.userManagement.listOrganizationMemberships({ @@ -693,7 +697,8 @@ const make = Effect.gen(function* () { userId, statuses: ["active", "pending"], }); - return response.data[0] ?? null; + const [membership] = response.data; + return membership === undefined ? null : membership; }), /** Get a user by ID. */ @@ -751,12 +756,6 @@ const make = Effect.gen(function* () { wos.userManagement.deleteOrganizationMembership(membershipId), ), - /** Get the role for a membership. */ - getOrgMembership: (membershipId: string) => - use("userManagement.getOrganizationMembership", (wos) => - wos.userManagement.getOrganizationMembership(membershipId), - ), - /** Update a membership's role. */ updateOrgMembershipRole: (membershipId: string, roleSlug: string) => use("userManagement.updateOrganizationMembership", (wos) => diff --git a/apps/cloud/src/db/schema.ts b/apps/cloud/src/db/schema.ts index 89e5c3cdb9..e0ecd2d664 100644 --- a/apps/cloud/src/db/schema.ts +++ b/apps/cloud/src/db/schema.ts @@ -82,15 +82,20 @@ export const organizations = pgTable( */ backfilledAt: timestamp("backfilled_at", { withTimezone: true }), /** - * When this organization was deleted, or null while it is live. Set by - * cloud's own deletion flow and by the `organization.deleted` event — - * which MINTS the row as a tombstone when the mirror has never seen the - * organization — and KEPT by the local purge (`db/org-deletion.ts`), - * which removes the organization's memberships and tenant data but - * leaves this row as a tombstone: a feeder that fetched a membership - * before the deletion and writes it after (a login that stalled across - * the deletion) finds the tombstone and does not mint the organization - * live. A marked organization is never renamed and authorizes nobody. + * When this organization was deleted, or null while it is live. Set FIRST + * by cloud's own deletion flow (`auth/handlers.ts` deleteOrganization), + * before the WorkOS delete and the local purge, and by the + * `organization.deleted` event for an org deleted in the WorkOS dashboard + * — which MINTS the row as a tombstone when the mirror has never seen the + * organization: membership is authorized from the local mirror, so a + * marked organization refuses every session at once, whether or not the + * later steps land. Membership rows are left as they are until the purge + * (so the admin who started the deletion can retry it after a step + * failed), and the purge (`db/org-deletion.ts`) removes them but KEEPS + * this row as a tombstone: a feeder that fetched a membership before the + * deletion and writes it after (a login that stalled across the deletion) + * finds the tombstone and does not mint the organization live. A marked + * organization is never renamed. */ deletedAt: timestamp("deleted_at", { withTimezone: true }), /** diff --git a/apps/cloud/src/extensions/billing/route.node.test.ts b/apps/cloud/src/extensions/billing/route.node.test.ts index 51a182b180..1f10d3abc5 100644 --- a/apps/cloud/src/extensions/billing/route.node.test.ts +++ b/apps/cloud/src/extensions/billing/route.node.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer } from "effect"; +import { MemberDirectory } from "@executor-js/api/server"; + import { UserStoreService } from "../../auth/context"; +import { MirrorReadiness, MirrorReadinessState } from "../../auth/mirror-readiness"; import { WorkOSClient, type WorkOSClientService } from "../../auth/workos"; +import { WorkOsMirror, type WorkOsMirrorShape } from "../../auth/workos-mirror"; import { resolveBillingOrganization } from "./route"; const createdAt = new Date("2026-01-01T00:00:00.000Z"); @@ -29,23 +33,43 @@ const stubWorkOS = Layer.succeed( WorkOSClient, new Proxy({} as WorkOSClientService, { get: (_target, prop) => { - if (prop === "listUserMemberships") { - return (userId: string) => - Effect.succeed({ - data: - userId === MEMBER - ? [ - { userId, organizationId: SESSION_ORG, status: "active" }, - { userId, organizationId: URL_ORG, status: "active" }, - ] - : [], - }); - } + // Membership is read from the mirror, never from WorkOS. return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); }, }), ); +// MEMBER is active in both orgs, as the mirror reports it. +// The mirror is READY in these tests (backfill complete, reconciler caught +// up), so membership is read from the stubbed directory, never from WorkOS. +const stubReadiness = Layer.succeed(MirrorReadiness)({ + state: () => Effect.succeed(MirrorReadinessState.Ready()), +}); + +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: (accountId, organizationId) => + Effect.succeed( + accountId === MEMBER && (organizationId === SESSION_ORG || organizationId === URL_ORG) + ? { + accountId, + membershipId: `om_${accountId}_${organizationId}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active" as const, + lastActiveAt: null, + } + : null, + ), + membershipById: () => Effect.die("billing auth does not look up by membership id"), + membershipsOf: () => Effect.die("billing auth reads one membership, not the list"), + members: () => Effect.die("billing auth does not list members"), + membersById: () => Effect.die("billing auth does not batch members"), + findByEmail: () => Effect.die("billing auth does not resolve emails"), +}); + const stubUsers = Layer.succeed(UserStoreService)({ use: (_op, fn) => Effect.promise(() => @@ -55,7 +79,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: org.id, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -64,7 +88,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ id, name: `Org ${id}`, slug: id, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, @@ -73,21 +97,34 @@ const stubUsers = Layer.succeed(UserStoreService)({ id: slug === URL_SLUG ? URL_ORG : "org_outsider", name: `Org ${slug}`, slug, - backfilledAt: null, + backfilledAt: createdAt, deletedAt: null, workosUpdatedAt: null, createdAt, }), + markOrganizationDeleted: async () => null, deleteOrganizationCascade: async () => {}, }), ), }); +// Authorization scans an organization the backfill never covered before it +// reads the mirror (`auth/organization.ts`); every org row above is marked +// backfilled, so the scan is never reached and the mirror is never written. +const stubMirror = Layer.succeed( + WorkOsMirror, + new Proxy({} as WorkOsMirrorShape, { + get: (_target, prop) => () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`), + }), +); + const run = (headers: Record) => resolveBillingOrganization( new Request("https://executor.test/api/billing/customer", { headers }), { userId: MEMBER }, - ).pipe(Effect.provide(Layer.mergeAll(stubWorkOS, stubUsers))); + ).pipe( + Effect.provide(Layer.mergeAll(stubWorkOS, stubUsers, stubDirectory, stubMirror, stubReadiness)), + ); describe("billing route org selector", () => { it.effect("fails closed when no selector header is sent", () => diff --git a/apps/cloud/src/extensions/billing/service.ts b/apps/cloud/src/extensions/billing/service.ts index c4759f269c..af04a5b829 100644 --- a/apps/cloud/src/extensions/billing/service.ts +++ b/apps/cloud/src/extensions/billing/service.ts @@ -58,6 +58,17 @@ const isCustomerNotFoundCause = (cause: unknown): boolean => { } }; +/** + * The HTTP status an Autumn failure carries, when the autumn-js SDK error + * underneath it has one; `undefined` for a network or SDK-level failure. + */ +export const autumnStatusOf = (failure: AutumnFailure): number | undefined => { + const cause = failure.cause; + if (typeof cause !== "object" || cause === null) return undefined; + const { statusCode } = cause as { readonly statusCode?: unknown }; + return typeof statusCode === "number" ? statusCode : undefined; +}; + // --------------------------------------------------------------------------- // Service interface // --------------------------------------------------------------------------- diff --git a/apps/cloud/src/extensions/routes.ts b/apps/cloud/src/extensions/routes.ts index 2f30bceaab..8d2b541d4b 100644 --- a/apps/cloud/src/extensions/routes.ts +++ b/apps/cloud/src/extensions/routes.ts @@ -31,6 +31,7 @@ import { AccountApi, AdminUsersApi } from "@executor-js/api"; import { requestScopedMiddleware, type MemberDirectory } from "@executor-js/api/server"; import { UserStoreService } from "../auth/context"; +import { MirrorReadiness } from "../auth/mirror-readiness"; import { WorkOsMirror } from "../auth/workos-mirror"; import { CloudAuthPublicHandlers, @@ -79,7 +80,9 @@ const spec = OpenApi.fromApi(CloudOpenApi); * core. */ export const makeCloudExtensionRoutes = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer< + DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness + >, ) => { // Session routes (login / callback / me / switch-org / …). Handlers yield // `UserStoreService` directly; the per-request DB combine keeps the postgres diff --git a/apps/cloud/src/mcp/agent-handler.ts b/apps/cloud/src/mcp/agent-handler.ts index 3d94d3e9cb..d10233953a 100644 --- a/apps/cloud/src/mcp/agent-handler.ts +++ b/apps/cloud/src/mcp/agent-handler.ts @@ -178,7 +178,7 @@ const propsForPrincipal = ( return { session: { organizationId: principal.organizationId, - // The org record the live membership check resolved microseconds ago, + // The org record the membership check resolved microseconds ago, // handed to the session DO so it never opens a connection of its own to // re-read it. An unnamed org (no auth plane could resolve one) is // omitted rather than sent empty, so the DO can tell "not carried" from diff --git a/apps/cloud/src/mcp/auth-provider.ts b/apps/cloud/src/mcp/auth-provider.ts index 054054a940..ab538e4ddb 100644 --- a/apps/cloud/src/mcp/auth-provider.ts +++ b/apps/cloud/src/mcp/auth-provider.ts @@ -91,7 +91,7 @@ const ORGANIZATION_AUTHORIZE_UNAVAILABLE = * Enrich a cloud {@link VerifiedToken} (which carries only accountId + * organizationId) into the full {@link Principal} the seam validates. * - * The org name and slug come from the record the live membership check just + * The org name and slug come from the record the membership check just * resolved — this is the whole point of `authorize` returning the record rather * than an id. They used to be dropped here (`organizationName: ""`), which left * the session Durable Object to re-read the same row over a fresh database @@ -182,8 +182,9 @@ export const cloudMcpAuthProviderLayer: Layer.Layer< // slug (`/acme/mcp`, what the install card prints) or a legacy org id // (`/org_xxx/mcp`), carried in the header by `prepareMcpOrgScope`; the // bare `/mcp` falls back to the token's `org_id`. Either way - // `orgAuth.authorize` resolves the selector and re-checks live WorkOS - // membership below, so the URL is a selector, not a trust boundary. + // `orgAuth.authorize` resolves the selector and re-checks membership + // against the local mirror below, so the URL is a selector, not a + // trust boundary. const organizationSelector = mcpOrganizationFromRequest(request) ?? token.organizationId; if (!organizationSelector) { yield* annotateMcpRequest(request, { token, parseBody }); diff --git a/apps/cloud/src/mcp/auth.ts b/apps/cloud/src/mcp/auth.ts index d14e6f4997..81ac8bc73f 100644 --- a/apps/cloud/src/mcp/auth.ts +++ b/apps/cloud/src/mcp/auth.ts @@ -17,6 +17,9 @@ import { ApiKeyService } from "../auth/api-keys"; import { BEARER_PREFIX } from "../auth/bearer"; import { authorizeOrganization } from "../auth/organization"; import { UserStoreService, makeUserStoreLayer } from "../auth/context"; +import { makeMemberDirectoryLayer } from "../auth/member-directory"; +import { makeMirrorReadinessLayer } from "../auth/mirror-readiness"; +import { makeWorkOsMirrorLayer } from "../auth/workos-mirror"; import { CoreSharedServices } from "../auth/workos"; import { makeDbLayer } from "../db/db"; import { bearerChallenge } from "./responses"; @@ -60,7 +63,7 @@ const TOOLKIT_SEGMENT = "/toolkits/"; // the token's `org_id` claim. start.ts / the test worker rewrite `/org_xxx/mcp` // (and the org-scoped discovery doc) to the bare path the shared envelope routes // and stash the URL-pinned org in this INTERNAL header; the provider reads it -// back. The org is re-checked against live WorkOS membership per request +// back. The org is re-checked against the local membership mirror per request // (`McpOrganizationAuth.authorize`), so the header — like the URL it came from — // is a SELECTOR, not a trust boundary. export const MCP_ORGANIZATION_HEADER = "x-executor-mcp-organization"; @@ -201,18 +204,28 @@ const verifyJwt = (token: string) => // `DbService.Live` would open its postgres socket on the first request and // illegally reuse it on later ones ("Cannot perform I/O on behalf of a // different request"), failing the org lookup on every follow-up — the -// "connected · tools fetch failed" symptom. A fresh DB + UserStore layer per -// call gives each request its own request-scoped socket. `CoreSharedServices` -// (WorkOS, no per-request socket) stays shared. +// "connected · tools fetch failed" symptom. A fresh DB + UserStore + +// MemberDirectory + WorkOsMirror layer per call gives each request its own +// request-scoped socket. `CoreSharedServices` (WorkOS, no per-request socket) stays shared. const makeMcpOrganizationAuthServices = () => { const dbLive = makeDbLayer(); const userStoreLive = makeUserStoreLayer().pipe(Layer.provide(dbLive)); - return Layer.mergeAll(dbLive, userStoreLive, CoreSharedServices); + const memberDirectoryLive = makeMemberDirectoryLayer().pipe(Layer.provide(dbLive)); + const mirrorReadinessLive = makeMirrorReadinessLayer().pipe(Layer.provide(dbLive)); + const workOsMirrorLive = makeWorkOsMirrorLayer().pipe(Layer.provide(dbLive)); + return Layer.mergeAll( + dbLive, + userStoreLive, + memberDirectoryLive, + mirrorReadinessLive, + workOsMirrorLive, + CoreSharedServices, + ); }; // A URL slug resolves through the mirror to its org id before the membership // check; an unknown slug authorizes nothing. Ids pass straight through — -// `authorizeOrganization` verifies live WorkOS membership either way. +// `authorizeOrganization` verifies membership against the mirror either way. const resolveOrgSelector = (selector: string) => selector.startsWith("org_") ? Effect.succeed(selector) diff --git a/apps/cloud/src/org/auth-middleware.ts b/apps/cloud/src/org/auth-middleware.ts index 477de037bc..d9f562494f 100644 --- a/apps/cloud/src/org/auth-middleware.ts +++ b/apps/cloud/src/org/auth-middleware.ts @@ -1,10 +1,16 @@ -import { Effect, Layer } from "effect"; +import { Context, Effect, Layer } from "effect"; import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; -import { AuthContext, requestScopedMiddleware } from "@executor-js/api/server"; +import { + AuthContext, + requestScopedMiddleware, + type MemberDirectory, +} from "@executor-js/api/server"; import { UserStoreService } from "../auth/context"; import { sessionFromSealed } from "../auth/middleware"; +import { MirrorReadiness } from "../auth/mirror-readiness"; +import { WorkOsMirror } from "../auth/workos-mirror"; import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../auth/organization"; import { WorkOSClient } from "../auth/workos"; import { DbService } from "../db/db"; @@ -27,7 +33,21 @@ const noOrganization = () => { status: 403 }, ); -const OrgAuthMiddleware = HttpRouter.middleware<{ provides: AuthContext }>()( +/** + * The caller's role in the session org, as `authorizeOrganizationSelector` + * read it for THIS request: from the mirror while the mirror is ready, from + * WorkOS otherwise (`auth/organization.ts`). Provided beside `AuthContext` — + * the shared seam, which carries no role — so the domain handlers' admin gate + * is this one value, never a second read of the mirror that would skip the + * readiness rule and admit a demoted admin on a stale row while the + * reconciler is behind. + */ +export class OrgMemberRole extends Context.Service< + OrgMemberRole, + { readonly memberRole: "admin" | "member" } +>()("@executor-js/cloud/OrgMemberRole") {} + +const OrgAuthMiddleware = HttpRouter.middleware<{ provides: AuthContext | OrgMemberRole }>()( Effect.gen(function* () { const captured = yield* Effect.context(); const workos = yield* WorkOSClient; @@ -62,10 +82,18 @@ const OrgAuthMiddleware = HttpRouter.middleware<{ provides: AuthContext }>()( roles: [], }); - return yield* Effect.provideService(httpEffect, AuthContext, auth); + return yield* Effect.provideContext( + httpEffect, + Context.make(AuthContext, auth).pipe( + Context.add(OrgMemberRole, { memberRole: org.memberRole }), + ), + ); }).pipe(Effect.provideContext(captured)); }), ); -export const orgAuthMiddleware = (rsLive: Layer.Layer) => - OrgAuthMiddleware.combine(requestScopedMiddleware(rsLive)).layer; +export const orgAuthMiddleware = ( + rsLive: Layer.Layer< + DbService | UserStoreService | MemberDirectory | MirrorReadiness | WorkOsMirror + >, +) => OrgAuthMiddleware.combine(requestScopedMiddleware(rsLive)).layer; diff --git a/apps/cloud/src/org/handlers.test.ts b/apps/cloud/src/org/handlers.test.ts index 05445a3e28..e1f4459cb2 100644 --- a/apps/cloud/src/org/handlers.test.ts +++ b/apps/cloud/src/org/handlers.test.ts @@ -1,24 +1,38 @@ -import { describe, it, expect } from "@effect/vitest"; +import { afterAll, describe, expect, it } from "@effect/vitest"; import { Data, Effect, Layer } from "effect"; +import { HttpRouter, HttpServer } from "effect/unstable/http"; +import { HttpApiBuilder } from "effect/unstable/httpapi"; -import { AuthContext } from "@executor-js/api/server"; +import { AuthContext, MemberDirectory, type DirectoryMember } from "@executor-js/api/server"; +import { UserStoreService } from "../auth/context"; +import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness"; +import { ORG_SELECTOR_HEADER } from "../auth/organization"; import { WorkOSClient, type WorkOSClientService } from "../auth/workos"; -import { Forbidden } from "./api"; +import { WorkOsMirror, type WorkOsMirrorShape } from "../auth/workos-mirror"; +import { DbService } from "../db/db"; +import { AutumnService } from "../extensions/billing/service"; +import { OrgHttpApi, Forbidden } from "./api"; +import { OrgMemberRole, orgAuthMiddleware } from "./auth-middleware"; +import { OrgHandlers, assertDomainInSessionOrg, requireAdmin } from "./handlers"; // --------------------------------------------------------------------------- // Domain-handler guards. The member / role / invite / org-name endpoints moved // to the shared WorkOS `AccountProvider` (covered by // `workos-account-service.test.ts`); this group now serves only the WorkOS // domain-verification endpoints. These tests pin the two guards those handlers -// share — `requireAdmin` and `assertDomainInSessionOrg` — which mirror -// `org/handlers.ts`. +// share — the REAL `requireAdmin` and `assertDomainInSessionOrg` exported from +// `org/handlers.ts`, so a change to the gate cannot pass on a stale copy — and +// the admin gate's SOURCE: the role `orgAuthMiddleware` resolved for the +// request, so a stale mirror row cannot admit a demoted admin while the mirror +// is not trusted (the readiness rule in `auth/organization.ts`). // --------------------------------------------------------------------------- // eslint-disable-next-line @typescript-eslint/no-explicit-any -- test stub needs wide function types type StubFn = (...args: never[]) => Effect.Effect; type StubOverrides = { - getUserOrgMembership?: StubFn; + authenticateSealedSession?: StubFn; + listUserMemberships?: StubFn; getOrganizationDomain?: StubFn; getOrganization?: StubFn; deleteOrganizationDomain?: StubFn; @@ -55,62 +69,27 @@ const adminAuth = { roles: [], }; -const memberAuth = { - accountId: "user_member", - organizationId: "org_1", - email: "member@test.com", - name: "Member", - avatarUrl: null, - roles: [], -}; - -const provide = (auth: typeof adminAuth, workosOverrides: StubOverrides = {}) => - Layer.mergeAll(Layer.succeed(AuthContext)(auth), stubWorkOS(workosOverrides)); - -// Mirrors `org/handlers.ts` `requireAdmin`. -const requireAdmin = Effect.gen(function* () { - const auth = yield* AuthContext; - if (auth.accountId === null) return yield* new Forbidden(); - const workos = yield* WorkOSClient; - const current = yield* workos.getUserOrgMembership(auth.organizationId, auth.accountId); - if (!current || current.role?.slug !== "admin") { - return yield* new Forbidden(); - } -}); - -const withCurrentMembership: StubOverrides = { - getUserOrgMembership: (_organizationId: string, userId: string) => - Effect.succeed( - userId === "user_admin" - ? { id: "mem_admin", userId, status: "active", role: { slug: "admin" } } - : { id: "mem_member", userId, status: "active", role: { slug: "member" } }, - ), -}; - -// Mirrors `org/handlers.ts` `assertDomainInSessionOrg`. -const assertDomainInSessionOrg = (domainId: string) => - Effect.gen(function* () { - const auth = yield* AuthContext; - const workos = yield* WorkOSClient; - const domain = yield* workos - .getOrganizationDomain(domainId) - .pipe(Effect.catchCause(() => Effect.succeed(null))); - if (!domain || domain.organizationId !== auth.organizationId) { - return yield* new Forbidden(); - } - }); +const provide = ( + memberRole: "admin" | "member", + workosOverrides: StubOverrides = {}, +): Layer.Layer => + Layer.mergeAll( + Layer.succeed(AuthContext)(adminAuth), + Layer.succeed(OrgMemberRole)({ memberRole }), + stubWorkOS(workosOverrides), + ); describe("Org domain handlers", () => { describe("requireAdmin", () => { it.effect("passes for an admin caller", () => - requireAdmin.pipe(Effect.provide(provide(adminAuth, withCurrentMembership))), + requireAdmin.pipe(Effect.provide(provide("admin"))), ); it.effect("rejects a non-admin caller with Forbidden", () => Effect.gen(function* () { const error = yield* Effect.flip(requireAdmin); expect(error).toBeInstanceOf(Forbidden); - }).pipe(Effect.provide(provide(memberAuth, withCurrentMembership))), + }).pipe(Effect.provide(provide("member"))), ); }); @@ -118,7 +97,7 @@ describe("Org domain handlers", () => { it.effect("passes when the domain belongs to the session org", () => assertDomainInSessionOrg("dom_1").pipe( Effect.provide( - provide(adminAuth, { + provide("admin", { getOrganizationDomain: () => Effect.succeed({ id: "dom_1", organizationId: "org_1", domain: "acme.test" }), }), @@ -132,7 +111,7 @@ describe("Org domain handlers", () => { expect(error).toBeInstanceOf(Forbidden); }).pipe( Effect.provide( - provide(adminAuth, { + provide("admin", { getOrganizationDomain: () => Effect.succeed({ id: "dom_other", organizationId: "org_2", domain: "evil.test" }), }), @@ -146,7 +125,7 @@ describe("Org domain handlers", () => { expect(error).toBeInstanceOf(Forbidden); }).pipe( Effect.provide( - provide(adminAuth, { + provide("admin", { getOrganizationDomain: () => Effect.fail(new UnstubbedWorkOSMethod({ method: "boom" })), }), ), @@ -154,3 +133,182 @@ describe("Org domain handlers", () => { ); }); }); + +// --------------------------------------------------------------------------- +// The admin gate over HTTP, through `orgAuthMiddleware`: the role the gate +// sees is the one the middleware resolved through `authorizeOrganizationSelector` +// — the mirror while it is ready, WorkOS otherwise. The mirror row below is +// STALE: it still says `admin` for a caller WorkOS has demoted to `member`. +// While the mirror is not trusted (the reconciler is behind), WorkOS's answer +// must decide, and the delete must be refused. +// --------------------------------------------------------------------------- + +const ORG = "org_1"; +const CALLER = "user_caller"; +const DOMAIN = "dom_1"; +const createdAt = new Date("2026-01-01T00:00:00.000Z"); + +// The mirror's row for the caller: an active admin — stale once WorkOS has +// demoted them and the reconciler has not landed the change yet. +const staleAdminRow: DirectoryMember = { + accountId: CALLER, + membershipId: `om_${CALLER}_${ORG}`, + organizationId: ORG, + email: null, + name: null, + avatarUrl: null, + role: "admin", + status: "active", + lastActiveAt: null, +}; + +const unread = (why: string) => () => Effect.die(why); +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: (accountId, organizationId) => + Effect.succeed(accountId === CALLER && organizationId === ORG ? staleAdminRow : null), + membershipById: unread("the org plane does not look up by membership id"), + membershipsOf: unread("the org plane reads one membership, not the list"), + members: unread("the org plane does not list members"), + membersById: unread("the org plane does not batch members"), + findByEmail: unread("the org plane does not resolve emails"), +}); + +const readiness = (state: MirrorReadinessState) => + Layer.succeed(MirrorReadiness)({ state: () => Effect.succeed(state) }); + +const organizationRow = (id: string) => ({ + id, + name: `Org ${id}`, + slug: id, + backfilledAt: createdAt, + deletedAt: null, + workosUpdatedAt: null, + createdAt, +}); + +// The store's operations are plain promises: an unexpected one defects +// through `Effect.promise`, the same way `unread` does for the services. +const unreadStore = (why: string) => () => Effect.runPromise(Effect.die(why)); +const stubUsers = Layer.succeed(UserStoreService)({ + use: (_op, fn) => + Effect.promise(() => + fn({ + ensureAccount: unreadStore("the org plane does not mint accounts"), + getAccount: unreadStore("the org plane does not read accounts"), + upsertOrganization: unreadStore("the org plane does not mirror organizations"), + getOrganization: async (id: string) => organizationRow(id), + getOrganizationBySlug: unreadStore("the selector below is an org id, not a slug"), + markOrganizationDeleted: unreadStore("the org plane does not delete organizations"), + deleteOrganizationCascade: unreadStore("the org plane does not delete organizations"), + }), + ), +}); + +// Authorization scans an organization the backfill never covered before it +// reads the mirror (`auth/organization.ts`); every org row above is marked +// backfilled, so the scan is never reached and the mirror is never written. +const stubMirror = Layer.succeed( + WorkOsMirror, + new Proxy({} as WorkOsMirrorShape, { + get: (_target, prop) => () => Effect.die(`unexpected WorkOsMirror.${String(prop)} call`), + }), +); + +// The handlers never reach the database here: the directory and the store are +// stubbed above, so the request-scoped `DbService` is a placeholder. +const stubDb = Layer.succeed(DbService)({ db: {} as never }); + +const stubAutumn = Layer.succeed(AutumnService)({ + use: unread("the delete does not consult billing"), + ensureCustomer: unread("the delete does not provision billing"), + checkExecutionBalance: unread("the delete does not check balances"), + trackExecution: unread("the delete does not track usage"), + setMemberSeats: unread("the delete does not count seats"), +}); + +// WorkOS as the org plane sees it: the caller's session, their CURRENT +// membership list (demoted to member), and the domain to delete. +const workosWithCallerAs = (role: "admin" | "member", deleted: string[]) => + stubWorkOS({ + authenticateSealedSession: () => + Effect.succeed({ userId: CALLER, email: "caller@placeholder.test", organizationId: ORG }), + listUserMemberships: () => + Effect.succeed({ + data: [ + { + id: staleAdminRow.membershipId, + organizationId: ORG, + status: "active", + role: { slug: role }, + }, + ], + }), + getOrganizationDomain: () => + Effect.succeed({ id: DOMAIN, organizationId: ORG, domain: "acme.test" }), + deleteOrganizationDomain: (domainId: string) => + Effect.sync(() => { + deleted.push(domainId); + }), + }); + +const orgApp = (state: MirrorReadinessState, workos: Layer.Layer) => { + const rsLive = Layer.mergeAll(stubDb, stubUsers, stubDirectory, stubMirror, readiness(state)); + const App = HttpApiBuilder.layer(OrgHttpApi).pipe( + Layer.provide(OrgHandlers), + Layer.provide(orgAuthMiddleware(rsLive)), + Layer.provide(workos), + Layer.provide(stubAutumn), + Layer.provide(HttpServer.layerServices), + ); + return HttpRouter.toWebHandler(App, { disableLogger: true }); +}; + +const apps: { dispose: () => Promise }[] = []; +afterAll(async () => { + await Promise.all(apps.map((app) => app.dispose())); +}); + +const deleteDomain = async (state: MirrorReadinessState, role: "admin" | "member") => { + const deleted: string[] = []; + const app = orgApp(state, workosWithCallerAs(role, deleted)); + apps.push(app); + const response = await app.handler( + new Request(`https://executor.test/org/domains/${DOMAIN}`, { + method: "DELETE", + headers: { cookie: "wos-session=sealed", [ORG_SELECTOR_HEADER]: ORG }, + }), + // beta.59: the handler type expects a context argument; this layer stack + // needs none at runtime — pass undefined like the api.request-scope tests. + undefined as never, + ); + return { status: response.status, deleted }; +}; + +describe("Org domain handlers over HTTP: the admin gate is the authorized role", () => { + it("lets a mirrored admin delete a domain while the mirror is ready", async () => { + const { status, deleted } = await deleteDomain(MirrorReadinessState.Ready(), "member"); + // WorkOS is not consulted for membership while the mirror is ready: the + // mirror row (admin) decides, and the demotion lands through the + // reconciler within its lag budget. + expect(status).toBe(200); + expect(deleted).toEqual([DOMAIN]); + }); + + it("refuses a demoted admin while the mirror is not ready, however stale the mirror row is", async () => { + const { status, deleted } = await deleteDomain( + MirrorReadinessState.ReconcilerStale({ drainedAt: null }), + "member", + ); + expect(status, "WorkOS says member; the stale admin row does not grant the delete").toBe(403); + expect(deleted).toEqual([]); + }); + + it("lets an admin WorkOS confirms delete a domain while the mirror is not ready", async () => { + const { status, deleted } = await deleteDomain( + MirrorReadinessState.ReconcilerStale({ drainedAt: null }), + "admin", + ); + expect(status).toBe(200); + expect(deleted).toEqual([DOMAIN]); + }); +}); diff --git a/apps/cloud/src/org/handlers.ts b/apps/cloud/src/org/handlers.ts index b338eee3f7..64c9f329b2 100644 --- a/apps/cloud/src/org/handlers.ts +++ b/apps/cloud/src/org/handlers.ts @@ -7,6 +7,7 @@ import { WorkOSClient } from "../auth/workos"; import { AutumnService } from "../extensions/billing/service"; import { resolveOrganization } from "../auth/organization"; import { Forbidden, OrgHttpApi } from "./api"; +import { OrgMemberRole } from "./auth-middleware"; // --------------------------------------------------------------------------- // Cloud-local org handlers — WorkOS domain-verification only. Members / roles / @@ -15,18 +16,21 @@ import { Forbidden, OrgHttpApi } from "./api"; // `OrgAuth` (org-scoped cookie session). // --------------------------------------------------------------------------- -const requireAdmin = Effect.gen(function* () { - const auth = yield* AuthContext; - // This plane is mounted behind the session-only `orgAuthMiddleware`, so the - // caller is always a member — but `AuthContext.accountId` is nullable for the - // platform credential, and membership of "no member" is not a question worth - // asking WorkOS. Refuse rather than assert. - if (auth.accountId === null) return yield* new Forbidden(); - const workos = yield* WorkOSClient; - const currentMembership = yield* workos.getUserOrgMembership(auth.organizationId, auth.accountId); - if (!currentMembership || currentMembership.role?.slug !== "admin") { - return yield* new Forbidden(); - } +/** + * The admin gate for the domain endpoints: the caller must be an `admin` of + * the session org. The role is the one `orgAuthMiddleware` resolved for this + * request through `authorizeOrganizationSelector` — an ACTIVE membership, + * read from the mirror only while the mirror is ready and from WorkOS + * otherwise — and is provided as `OrgMemberRole`. The gate is that one value, + * as on the sibling gates (`workos-account-service.ts` `requireAdmin`, + * `admin-users-api.ts` `authorizeTenant`): a second read of the mirror here + * would skip the readiness rule, and while the reconciler is behind a stale + * row would keep admitting an admin demoted in the WorkOS dashboard. Fails + * with `Forbidden` for a member. Exported for its test only. + */ +export const requireAdmin = Effect.gen(function* () { + const { memberRole } = yield* OrgMemberRole; + if (memberRole !== "admin") return yield* new Forbidden(); }); // Target-ownership check — independent of caller privilege. `requireAdmin` @@ -37,7 +41,8 @@ const requireAdmin = Effect.gen(function* () { // workspace API key is workspace-wide and WorkOS does not enforce per-org // ownership on delete by id. Failures (not found OR org mismatch) both surface // as Forbidden so we don't leak existence of ids outside the caller's org. -const assertDomainInSessionOrg = (domainId: string) => +// Exported for its test only. +export const assertDomainInSessionOrg = (domainId: string) => Effect.gen(function* () { const auth = yield* AuthContext; const workos = yield* WorkOSClient; diff --git a/apps/host-selfhost/src/auth/member-directory.test.ts b/apps/host-selfhost/src/auth/member-directory.test.ts index 1671adb8da..bd25e8d13b 100644 --- a/apps/host-selfhost/src/auth/member-directory.test.ts +++ b/apps/host-selfhost/src/auth/member-directory.test.ts @@ -114,10 +114,18 @@ describe("self-host MemberDirectory", () => { const result = await run( Effect.gen(function* () { const d = yield* MemberDirectory; + const one = yield* d.membership(grace, organizationId); + const graceRow = one?.membershipId ?? "member_missing"; return { - one: yield* d.membership(grace, organizationId), + one, oneInactive: yield* d.membership(grace, organizationId, ["inactive"]), none: yield* d.membership(outsider.user.id, organizationId), + byId: yield* d.membershipById(organizationId, graceRow), + byIdForeign: yield* d.membershipById("org_other", graceRow), + byIdUnknown: yield* d.membershipById(organizationId, "member_unknown"), + ofGrace: yield* d.membershipsOf(grace), + ofOutsider: yield* d.membershipsOf(outsider.user.id), + ofGraceInactive: yield* d.membershipsOf(grace, ["inactive"]), batch: yield* d.membersById(organizationId, [ada, linus, outsider.user.id, "nobody"]), byEmail: yield* d.findByEmail(organizationId, "ada.lovelace@placeholder.test"), unknown: yield* d.findByEmail(organizationId, "outsider@placeholder.test"), @@ -127,6 +135,15 @@ describe("self-host MemberDirectory", () => { expect(result.one?.role).toBe("member"); expect(result.oneInactive, "Better Auth members are always active").toBeNull(); expect(result.none).toBeNull(); + expect(result.byId?.accountId).toBe(grace); + expect(result.byIdForeign, "the member row id is scoped to its org").toBeNull(); + expect(result.byIdUnknown).toBeNull(); + expect( + result.ofGrace.map((m) => m.organizationId), + "the single org", + ).toEqual([organizationId]); + expect(result.ofOutsider).toEqual([]); + expect(result.ofGraceInactive, "Better Auth members are always active").toEqual([]); expect([...result.batch.keys()].sort()).toEqual([ada, linus].sort()); expect(result.byEmail?.accountId).toBe(ada); expect(result.unknown, "a user with no membership is not a member").toBeNull(); diff --git a/apps/host-selfhost/src/auth/member-directory.ts b/apps/host-selfhost/src/auth/member-directory.ts index 4c84841174..f6787ac3e6 100644 --- a/apps/host-selfhost/src/auth/member-directory.ts +++ b/apps/host-selfhost/src/auth/member-directory.ts @@ -170,6 +170,29 @@ const makeService = (adapter: BetterAuthAdapter): MemberDirectoryShape => { { field: "organizationId", value: organizationId }, ]).pipe(Effect.map((members) => members[0] ?? null)), + membershipById: (organizationId, membershipId) => + load("membershipById", [ + { field: "id", value: membershipId }, + { field: "organizationId", value: organizationId }, + ]).pipe(Effect.map((members) => members[0] ?? null)), + + membershipsOf: (accountId, statuses = DEFAULT_MEMBER_STATUSES) => + // Every Better Auth member is active; a query for other statuses only + // has nothing to report. + !statuses.includes("active") + ? Effect.succeed([]) + : load("membershipsOf", [{ field: "userId", value: accountId }]).pipe( + Effect.map((members) => + [...members].sort((a, b) => + a.organizationId < b.organizationId + ? -1 + : a.organizationId > b.organizationId + ? 1 + : 0, + ), + ), + ), + members: (organizationId, query: MemberQuery = {}) => Effect.gen(function* () { if (!reportsActive(query.statuses ?? DEFAULT_MEMBER_STATUSES)) return []; diff --git a/packages/core/api/src/server/member-directory.ts b/packages/core/api/src/server/member-directory.ts index 50bef4a342..a80e1adf68 100644 --- a/packages/core/api/src/server/member-directory.ts +++ b/packages/core/api/src/server/member-directory.ts @@ -80,6 +80,26 @@ export interface MemberDirectoryShape { organizationId: string, statuses?: readonly MemberStatus[], ) => Effect.Effect; + /** + * One membership by its host membership ROW id, any status; `null` when + * THIS org holds no such row. The ownership gate for host-specific writes + * (remove, change role): an id leaked from another org resolves to `null` + * here, so a point read answers "is this ours" without listing the org. + */ + readonly membershipById: ( + organizationId: string, + membershipId: string, + ) => Effect.Effect; + /** + * Every organization membership one account holds, across organizations — + * the org switcher's list and the per-user organization limit. `statuses` + * defaults to active + pending; ordered by `organizationId` so the answer + * is stable. One read for the whole set, never a lookup per org. + */ + readonly membershipsOf: ( + accountId: string, + statuses?: readonly MemberStatus[], + ) => Effect.Effect; /** The org's members matching `query` (see {@link MemberQuery} for defaults). */ readonly members: ( organizationId: string,