Authorize org membership from the local mirror

.changeset/member-directory-auth-cutover.md

@@ -0,0 +1,11 @@
+---
+"@executor-js/cloud": patch
+"@executor-js/api": patch
+"@executor-js/host-selfhost": patch
+---
+
+Cloud now authorizes every protected request against the local membership mirror through the shared `MemberDirectory` seam: the per-request org membership check, the admin gates on the account and admin planes, the org switcher's organization list, and the free-organization limit all read the mirror instead of calling WorkOS. WorkOS is now a write target and an event source only. The seam gains `membershipsOf(accountId)` and `membershipById(organizationId, membershipId)` on both hosts.
+
+The mirror is trusted only while it is **ready**: the backfill has written every organization and the Events reconciler has drained the stream within the last ten minutes (both recorded on the `workos_sync` row). Until then the membership check falls back to WorkOS, exactly as before, so a member the backfill has not written yet is not locked out and a member revoked while the reconciler was down is not let in. The deploy runs `scripts/ensure-workos-mirror-ready.ts` after the migrations: it runs the backfill if needed, drains the events stream itself if the reconciler has not recently (so the gate never waits on a cron this same deploy ships), and fails the deploy if the mirror is still not ready. An organization the mirror does not hold at all (one that predates the mirror and nobody has signed in to since) is resolved from WorkOS on demand for a caller WorkOS confirms as its member, so CLI and MCP tokens naming such an organization are not refused. Deleting an organization now cancels billing before deleting the WorkOS organization, and a retry after a partial deletion is admitted from the mirror even while the mirror is not ready.
+
+**Ops step (cloud):** add the `WORKOS_API_KEY` secret to the `production` GitHub environment so the deploy gate can run the backfill.

.claude/skills/prod-telemetry/SKILL.md

@@ -57,6 +57,23 @@ join the same traces via traceparent).
   `execute`/`execute-action` calls `mcp.execute.code` (the script itself,
   capped at 10k chars — cloud-only content capture; local/self-host
   telemetry never records content).
+- `auth.authorize_organization` — every membership authorization.
+  `mirror.ready` (bool: the local membership mirror answered; `false` =
+  the request fell back to a live WorkOS read) and `mirror.readiness`
+  (why: `ready`, `backfill pending: …`, `reconciler stale: …`). The
+  mirror's write spans are `workos_mirror.<op>`; the reconciler run is
+  `workos_events.sync`. `workos_sync.drained_at` in the prod DB is the
+  reconciler heartbeat.
+
+**Recipe — membership-mirror fallback rate (should be ~0 after cutover):**
+
+```apl
+['executor-cloud']
+| where _time > ago(1h) and name == "auth.authorize_organization"
+| extend ready = tobool(['attributes.custom']['mirror.ready'])
+| extend why = tostring(['attributes.custom']['mirror.readiness'])
+| summarize n = count() by ready, why
+```
 
 **Recipe — error signatures by class (the daily-digest query):**
 

.github/workflows/deploy.yml

@@ -54,6 +54,19 @@ jobs:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
 
+      # The build below authorizes every request from the local membership
+      # mirror. This runs the mirror backfill if it has not completed, drains
+      # the WorkOS events stream itself if the reconciler has not recently
+      # (it does not wait on the cron, which this same deploy may be the one
+      # to ship), and FAILS the deploy if the mirror is still not ready — see
+      # scripts/ensure-workos-mirror-ready.ts.
+      - name: Backfill and verify the membership mirror
+        run: bun run scripts/ensure-workos-mirror-ready.ts
+        working-directory: apps/cloud
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          WORKOS_API_KEY: ${{ secrets.WORKOS_API_KEY }}
+
   deploy-cloud:
     name: Deploy cloud
     runs-on: blacksmith-4vcpu-ubuntu-2404

apps/cloud/package.json

@@ -36,6 +36,7 @@
     "db:backfill-workos-mirror:dev": "op run --env-file=.env.op -- bun run scripts/backfill-workos-mirror.ts",
     "db:drain-workos-events:prod": "op run --env-file=.env.production -- bun run scripts/drain-workos-events.ts",
     "db:drain-workos-events:dev": "op run --env-file=.env.op -- bun run scripts/drain-workos-events.ts",
+    "db:ensure-workos-mirror-ready:prod": "op run --env-file=.env.production -- bun run scripts/ensure-workos-mirror-ready.ts",
     "routes:gen": "bun scripts/gen-routes.ts",
     "vendor-wasm": "bun run scripts/vendor-quickjs-wasm.ts"
   },

apps/cloud/scripts/ensure-workos-mirror-ready.ts

@@ -0,0 +1,115 @@
+/* oxlint-disable executor/no-try-catch-or-throw -- boundary: out-of-band deploy gate over a raw postgres connection */
+// ---------------------------------------------------------------------------
+// Deploy gate: make the membership mirror READY before the build that
+// authorizes from it goes live, and fail the deploy if it cannot be.
+//
+//   bun run db:ensure-workos-mirror-ready:prod   # op run --env-file=.env.production
+//   (deploy.yml runs it after the migrations, before the cloud deploy)
+//
+// Readiness is the SAME rule the request path applies
+// (`src/auth/mirror-readiness-store.ts`): the one-off backfill has written
+// every organization (`workos_sync.backfill_completed_at`) AND the events
+// reconciler has drained the stream within its lag budget
+// (`workos_sync.drained_at`). Until both hold the deployed build reads
+// membership from WorkOS instead of the mirror, so an unready mirror never
+// locks anyone out or lets a revoked member in — but a deploy that leaves it
+// unready would run every request through that fallback, which is the state
+// this whole cutover exists to leave behind. So this gate:
+//   1. reads the readiness row;
+//   2. if the backfill has not completed, RUNS it (scripts/backfill-workos-mirror.ts,
+//      idempotent) and reads again;
+//   3. if the reconciler has not drained recently, DRAINS the stream itself
+//      (scripts/drain-workos-events.ts: the same replay the Worker's cron
+//      runs, over this connection) and reads again — never merely waits for
+//      the cron: this gate runs BEFORE the build that carries the cron may
+//      have been deployed, and a gate that only waited could not pass until
+//      the reconciler build had shipped on its own, by hand. A cron that is
+//      already live is safe beside it (the cursor's compare-and-set gives
+//      the stream one owner at a time);
+//   4. exits 0 only when the mirror is ready, and 1 with the reason otherwise.
+// Needs DATABASE_URL and WORKOS_API_KEY (the backfill and the drain read WorkOS).
+// ---------------------------------------------------------------------------
+
+import { spawnSync } from "node:child_process";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { drizzle } from "drizzle-orm/postgres-js";
+import postgres from "postgres";
+
+import {
+  MirrorReadinessState,
+  describeMirrorReadiness,
+  readMirrorReadiness,
+} from "../src/auth/mirror-readiness-store";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const BACKFILL_SCRIPT = resolve(__dirname, "backfill-workos-mirror.ts");
+const DRAIN_SCRIPT = resolve(__dirname, "drain-workos-events.ts");
+
+const connectionString = process.env.DATABASE_URL;
+if (!connectionString) {
+  console.error("DATABASE_URL is not set");
+  process.exit(1);
+}
+
+const usesLocalDatabase =
+  connectionString.includes("127.0.0.1") || connectionString.includes("localhost");
+
+const sql = postgres(connectionString, {
+  max: 1,
+  prepare: false,
+  ...(usesLocalDatabase ? {} : { ssl: "require" as const }),
+});
+const db = drizzle(sql);
+
+const log = (line: string) => console.log(`[mirror-ready] ${line}`);
+
+const readiness = () => readMirrorReadiness(db, new Date());
+
+// The backfill and drain scripts own their own WorkOS + database wiring;
+// running them as subprocesses (with this process's env) keeps that wiring
+// in one place.
+const runScript = (what: string, script: string) => {
+  if (!process.env.WORKOS_API_KEY) {
+    throw new Error(`WORKOS_API_KEY is not set; the mirror ${what} cannot run`);
+  }
+  const result = spawnSync("bun", ["run", script], {
+    stdio: "inherit",
+    env: process.env,
+  });
+  if (result.status !== 0) {
+    throw new Error(`the mirror ${what} exited with status ${result.status ?? "unknown"}`);
+  }
+};
+
+try {
+  let state = await readiness();
+  log(describeMirrorReadiness(state));
+
+  if (MirrorReadinessState.$is("BackfillPending")(state)) {
+    log("backfill not completed; running scripts/backfill-workos-mirror.ts");
+    runScript("backfill", BACKFILL_SCRIPT);
+    state = await readiness();
+    log(describeMirrorReadiness(state));
+  }
+
+  if (MirrorReadinessState.$is("ReconcilerStale")(state)) {
+    log("events stream not drained recently; running scripts/drain-workos-events.ts");
+    runScript("drain", DRAIN_SCRIPT);
+    state = await readiness();
+    log(describeMirrorReadiness(state));
+  }
+
+  if (!MirrorReadinessState.$is("Ready")(state)) {
+    console.error(
+      `[mirror-ready] the membership mirror is not ready: ${describeMirrorReadiness(state)}. ` +
+        "The deployed build would read membership from WorkOS on every request until it is. " +
+        "Check that WorkOS is reachable and the backfill has run, then rerun the deploy.",
+    );
+    process.exit(1);
+  }
+  log("the membership mirror is ready");
+} finally {
+  await sql.end({ timeout: 5 });
+}

apps/cloud/src/account/account-api.ts

@@ -10,6 +10,7 @@ import {
 
 import { ApiKeyService } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
+import { MirrorReadiness } from "../auth/mirror-readiness";
 import { WorkOsMirror } from "../auth/workos-mirror";
 import { sessionFromSealed, type Session } from "../auth/middleware";
 import { WorkOSClient } from "../auth/workos";
@@ -99,11 +100,15 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi
  * (the seat-gate) stays a residual requirement, satisfied by the app `boot`.
  */
 export const workosAccountMiddleware = (
-  rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror | MemberDirectory>,
+  rsLive: Layer.Layer<
+    DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness
+  >,
 ) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer;
 
 export const makeAccountApiLive = (
-  rsLive: Layer.Layer<DbService | UserStoreService | WorkOsMirror | MemberDirectory>,
+  rsLive: Layer.Layer<
+    DbService | UserStoreService | WorkOsMirror | MemberDirectory | MirrorReadiness
+  >,
 ) => {
   // Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it
   // closes over the per-request postgres socket), so it can't be a self-

apps/cloud/src/account/org-api-key-revoke.node.test.ts

@@ -6,6 +6,7 @@ import { AccountError, AccountForbidden } from "@executor-js/api";
 
 import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
+import { MirrorReadiness, MirrorReadinessState } from "../auth/mirror-readiness";
 import { ORG_SELECTOR_HEADER } from "../auth/organization";
 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
 import { WorkOsMirror } from "../auth/workos-mirror";
@@ -63,32 +64,12 @@ const session = (accountId: string) => ({
   refreshedSession: null,
 });
 
-/** Membership roles: only ADMIN carries the `admin` role slug. */
+// Membership is read from the mirror, never from WorkOS: revoke makes no
+// WorkOS call at all.
 const stubWorkOS = Layer.succeed(
   WorkOSClient,
   new Proxy({} as WorkOSClientService, {
-    get: (_target, prop) => {
-      if (prop === "listUserMemberships") {
-        return (userId: string) =>
-          Effect.succeed({
-            data: [{ userId, organizationId: ORG, status: "active" }],
-          });
-      }
-      if (prop === "getUserOrgMembership") {
-        return (organizationId: string, userId: string) =>
-          Effect.succeed(
-            organizationId === ORG
-              ? {
-                  id: `om_${userId}`,
-                  userId,
-                  organizationId,
-                  role: { slug: userId === ADMIN ? "admin" : "member" },
-                }
-              : null,
-          );
-      }
-      return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`);
-    },
+    get: (_target, prop) => () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`),
   }),
 );
 
@@ -101,7 +82,7 @@ const stubUsers = Layer.succeed(UserStoreService)({
         upsertOrganization: async (org: { id: string; name: string }) => ({
           ...org,
           slug: org.id,
-          backfilledAt: null,
+          backfilledAt: createdAt,
           deletedAt: null,
           workosUpdatedAt: null,
           createdAt,
@@ -110,7 +91,7 @@ const stubUsers = Layer.succeed(UserStoreService)({
           id,
           name: `Org ${id}`,
           slug: id,
-          backfilledAt: null,
+          backfilledAt: createdAt,
           deletedAt: null,
           workosUpdatedAt: null,
           createdAt,
@@ -119,11 +100,12 @@ const stubUsers = Layer.succeed(UserStoreService)({
           id: slug,
           name: `Org ${slug}`,
           slug,
-          backfilledAt: null,
+          backfilledAt: createdAt,
           deletedAt: null,
           workosUpdatedAt: null,
           createdAt,
         }),
+        markOrganizationDeleted: async () => null,
         deleteOrganizationCascade: async () => {},
       }),
     ),
@@ -147,12 +129,37 @@ const stubMirror = Layer.succeed(WorkOsMirror)({
   organizationBackfilledAt: () => Effect.die("revoke does not report seats"),
 });
 
-// Revoke lists no members either.
+// The mirror as the directory reads it: both are active members of ORG, and
+// only ADMIN carries the `admin` role. Revoke reads the caller's membership
+// (the org check and the admin gate) and nothing else.
+// The mirror is READY in these tests (backfill complete, reconciler caught
+// up), so membership is read from the stubbed directory, never from WorkOS.
+const stubReadiness = Layer.succeed(MirrorReadiness)({
+  state: () => Effect.succeed(MirrorReadinessState.Ready()),
+});
+
 const stubDirectory = Layer.succeed(MemberDirectory)({
-  membership: () => Effect.die("revoke does not read the member directory"),
-  members: () => Effect.die("revoke does not read the member directory"),
-  membersById: () => Effect.die("revoke does not read the member directory"),
-  findByEmail: () => Effect.die("revoke does not read the member directory"),
+  membership: (accountId, organizationId) =>
+    Effect.succeed(
+      organizationId === ORG
+        ? {
+            accountId,
+            membershipId: `om_${accountId}`,
+            organizationId,
+            email: null,
+            name: null,
+            avatarUrl: null,
+            role: accountId === ADMIN ? "admin" : "member",
+            status: "active" as const,
+            lastActiveAt: null,
+          }
+        : null,
+    ),
+  membershipById: () => Effect.die("revoke does not look up by membership id"),
+  membershipsOf: () => Effect.die("revoke does not list the caller's memberships"),
+  members: () => Effect.die("revoke does not list members"),
+  membersById: () => Effect.die("revoke does not batch members"),
+  findByEmail: () => Effect.die("revoke does not resolve emails"),
 });
 
 const stubAutumn = Layer.succeed(AutumnService)({
@@ -194,6 +201,7 @@ const providerWith = (accountId: string) => {
             stubUsers,
             stubMirror,
             stubDirectory,
+            stubReadiness,
             stubApiKeys,
             stubAutumn,
             Layer.succeed(AccountCaller)({ session: session(accountId) }),