diff --git a/apps/cloud/drizzle/0019_workos_mirror_sync_state.sql b/apps/cloud/drizzle/0019_workos_mirror_sync_state.sql new file mode 100644 index 0000000000..419595746a --- /dev/null +++ b/apps/cloud/drizzle/0019_workos_mirror_sync_state.sql @@ -0,0 +1,21 @@ +-- Backfill completeness per organization (`organizations.backfilled_at`: when +-- its membership list was last fully scanned from WorkOS), the organization +-- tombstone (`organizations.deleted_at`: kept by the local purge so a delayed +-- login cannot re-mint a deleted organization), the organization name stamp +-- (`organizations.workos_updated_at`: a name write stamped earlier is refused, +-- so a delayed login cannot revert a rename), and on the "events" row of +-- `workos_sync` the Events API replay boundary (`range_start`) the +-- reconciler's first run reads from plus the backfill completion mark +-- (`backfill_completed_at`) the authorization path checks before it trusts +-- the mirror over WorkOS. A database with no organizations has nothing to +-- backfill, so seed both there (fresh dev, test, and e2e databases); a +-- database that already holds organizations gets them from the backfill +-- script (scripts/backfill-workos-mirror.ts). +ALTER TABLE "organizations" ADD COLUMN "backfilled_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "organizations" ADD COLUMN "deleted_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "organizations" ADD COLUMN "workos_updated_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "workos_sync" ADD COLUMN "range_start" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "workos_sync" ADD COLUMN "backfill_completed_at" timestamp with time zone;--> statement-breakpoint +INSERT INTO "workos_sync" ("id", "cursor", "range_start", "backfill_completed_at", "updated_at") +SELECT 'events', NULL, now(), now(), now() +WHERE NOT EXISTS (SELECT 1 FROM "organizations"); diff --git a/apps/cloud/drizzle/meta/0019_snapshot.json b/apps/cloud/drizzle/meta/0019_snapshot.json new file mode 100644 index 0000000000..eed3fb2107 --- /dev/null +++ b/apps/cloud/drizzle/meta/0019_snapshot.json @@ -0,0 +1,1754 @@ +{ + "id": "26e5a445-9146-40bb-afaf-0f26bfb00818", + "prevId": "fe5ddc71-31c5-4144-a879-11ea71a63735", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sign_in_at": { + "name": "last_sign_in_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_email_lower_idx": { + "name": "accounts_email_lower_idx", + "columns": [ + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.membership_tombstones": { + "name": "membership_tombstones", + "schema": "", + "columns": { + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "membership_tombstones_organization_id_idx": { + "name": "membership_tombstones_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "membership_tombstones_account_id_accounts_id_fk": { + "name": "membership_tombstones_account_id_accounts_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "membership_tombstones_organization_id_organizations_id_fk": { + "name": "membership_tombstones_organization_id_organizations_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_membership_id_unique": { + "name": "memberships_membership_id_unique", + "columns": [ + { + "expression": "membership_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_organization_id_idx": { + "name": "memberships_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backfilled_at": { + "name": "backfilled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workos_sync": { + "name": "workos_sync", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "range_start": { + "name": "range_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_completed_at": { + "name": "backfill_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index 421c19a0ab..b629ea283f 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -134,6 +134,13 @@ "when": 1789570778982, "tag": "0018_member_directory_mirror", "breakpoints": true + }, + { + "idx": 19, + "version": "7", + "when": 1789571259533, + "tag": "0019_workos_mirror_sync_state", + "breakpoints": true } ] } diff --git a/apps/cloud/package.json b/apps/cloud/package.json index 4feae6f1ac..0c93031685 100644 --- a/apps/cloud/package.json +++ b/apps/cloud/package.json @@ -32,6 +32,8 @@ "db:backfill-org-slugs:dev": "op run --env-file=.env.op -- bun run scripts/backfill-org-slugs.ts", "db:backfill-subjects:prod": "op run --env-file=.env.production -- bun run scripts/backfill-subjects.ts", "db:backfill-subjects:dev": "op run --env-file=.env.op -- bun run scripts/backfill-subjects.ts", + "db:backfill-workos-mirror:prod": "op run --env-file=.env.production -- bun run scripts/backfill-workos-mirror.ts", + "db:backfill-workos-mirror:dev": "op run --env-file=.env.op -- bun run scripts/backfill-workos-mirror.ts", "routes:gen": "bun scripts/gen-routes.ts", "vendor-wasm": "bun run scripts/vendor-quickjs-wasm.ts" }, diff --git a/apps/cloud/scripts/backfill-workos-mirror.ts b/apps/cloud/scripts/backfill-workos-mirror.ts new file mode 100644 index 0000000000..712138585d --- /dev/null +++ b/apps/cloud/scripts/backfill-workos-mirror.ts @@ -0,0 +1,102 @@ +// --------------------------------------------------------------------------- +// One-off data backfill: fill the membership mirror (`accounts` profile +// columns + `memberships` rows, migration 0018) from WorkOS for every +// organization the mirror already knows. +// +// bun run db:backfill-workos-mirror:prod # op run --env-file=.env.production +// bun run db:backfill-workos-mirror:dev # against the local PGlite dev db +// +// For each live row in `organizations`: list EVERY membership WorkOS holds +// for it (active, pending, and inactive — a listing that skipped inactive +// ones would have the scan tombstone them, see `auth/workos-mirror-backfill.ts`), +// fetch each member's user (concurrency 5), and apply the listing in one +// transaction through the same guarded store the request path +// uses (`auth/workos-mirror-store.ts`): upsert user + membership, tombstone +// any mirrored membership of that org WorkOS no longer lists, and mark the +// org backfilled (`organizations.backfilled_at`) — the per-org mark the seat +// gates check before trusting a count from the mirror; an org left unmarked +// is scanned on demand the first time its seats are counted. Idempotent — +// the upserts refuse anything older than the stored WorkOS `updatedAt`, so +// re-running is safe, never rewinds a fresher row, and repairs a stale one; +// and a listing older than one already applied (two runs overlapping) is +// refused whole, so it cannot resurrect a membership the later listing found +// gone. Pass --dry-run to read and count without writing. +// +// DEPLOY ORDER: run this against production BEFORE deploying the builds that +// reconcile from the Events API and read seat counts from the mirror, so no +// request pays for an on-demand scan. The FIRST completed run records the +// events replay boundary (the reconciler's first run reads from it; without +// one it waits); later runs keep it, since only the events stream covers the +// org renames and user deletions between two runs. A run that fails part-way +// keeps the marks of the orgs it finished, records no boundary, and is safe +// to repeat. Verify the printed membership count against the WorkOS +// dashboard. +// --------------------------------------------------------------------------- + +import { asc, isNull } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { Effect } from "effect"; +import postgres from "postgres"; +import { WorkOS } from "@workos-inc/node"; + +import { backfillWorkOsMirror } from "../src/auth/workos-mirror-backfill"; +import { makeWorkOsMirrorStore } from "../src/auth/workos-mirror-store"; +import { organizations } from "../src/db/schema"; + +const dryRun = process.argv.includes("--dry-run"); + +const connectionString = process.env.DATABASE_URL; +if (!connectionString) { + console.error("DATABASE_URL is not set"); + process.exit(1); +} +const apiKey = process.env.WORKOS_API_KEY; +if (!apiKey) { + console.error("WORKOS_API_KEY is not set"); + process.exit(1); +} + +const usesLocalDatabase = + connectionString.includes("127.0.0.1") || connectionString.includes("localhost"); + +const sql = postgres(connectionString, { + max: 1, + prepare: false, + ...(usesLocalDatabase ? {} : { ssl: "require" as const }), +}); +const db = drizzle(sql); +const workos = new WorkOS(apiKey); + +// The script boundary: raw SDK / driver promises lifted once, here. +const fromPromise = (fn: () => Promise) => + Effect.tryPromise({ try: fn, catch: (cause) => cause }); + +await Effect.runPromise( + backfillWorkOsMirror( + { + listOrganizationIds: () => + fromPromise(async () => { + // Never a deleted organization: its row is a tombstone (its + // memberships are purged, WorkOS no longer has it) and the mirror + // refuses a scan of it anyway. + const rows = await db + .select({ id: organizations.id }) + .from(organizations) + .where(isNull(organizations.deletedAt)) + .orderBy(asc(organizations.createdAt)); + return rows.map((row) => row.id); + }), + listOrgMembers: (organizationId) => + fromPromise(async () => { + const page = await workos.userManagement.listOrganizationMemberships({ + organizationId, + statuses: ["active", "pending", "inactive"], + }); + return page.listMetadata.after ? page.autoPagination() : page.data; + }), + getUser: (userId) => fromPromise(() => workos.userManagement.getUser(userId)), + }, + makeWorkOsMirrorStore(db), + { dryRun, log: (line) => console.log(line) }, + ).pipe(Effect.ensuring(Effect.promise(() => sql.end({ timeout: 5 })))), +); diff --git a/apps/cloud/scripts/test-globalsetup.ts b/apps/cloud/scripts/test-globalsetup.ts index 7efc25afe0..1bdb1cd9a5 100644 --- a/apps/cloud/scripts/test-globalsetup.ts +++ b/apps/cloud/scripts/test-globalsetup.ts @@ -42,7 +42,11 @@ export default async function setup() { db = await PGlite.create(); await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER }); - server = new PGLiteSocketServer({ db, port: PORT, host: "127.0.0.1" }); + // PGlite is single-session; pglite-socket multiplexes connections onto it + // by queueing whole transactions. Two connections let a test open two + // transactions and interleave them (the mirror's scan-vs-feeder race in + // auth/workos-mirror.node.test.ts); a third would be refused at once. + server = new PGLiteSocketServer({ db, port: PORT, host: "127.0.0.1", maxConnections: 2 }); await server.start(); // eslint-disable-next-line no-console diff --git a/apps/cloud/src/account/account-api.ts b/apps/cloud/src/account/account-api.ts index a67d6f4f6d..1e8075c866 100644 --- a/apps/cloud/src/account/account-api.ts +++ b/apps/cloud/src/account/account-api.ts @@ -9,6 +9,7 @@ import { import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; +import { WorkOsMirror } from "../auth/workos-mirror"; import { sessionFromSealed, type Session } from "../auth/middleware"; import { WorkOSClient } from "../auth/workos"; import { AutumnService } from "../extensions/billing/service"; @@ -95,10 +96,13 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi * account service closes over the per-request postgres socket). `AutumnService` * (the seat-gate) stays a residual requirement, satisfied by the app `boot`. */ -export const workosAccountMiddleware = (rsLive: Layer.Layer) => - AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer; +export const workosAccountMiddleware = ( + rsLive: Layer.Layer, +) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer; -export const makeAccountApiLive = (rsLive: Layer.Layer) => { +export const makeAccountApiLive = ( + rsLive: Layer.Layer, +) => { // Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it // closes over the per-request postgres socket), so it can't be a self- // contained `Layer` — it combines its own middleware with diff --git a/apps/cloud/src/account/org-api-key-revoke.node.test.ts b/apps/cloud/src/account/org-api-key-revoke.node.test.ts index 736c1adafa..6682280a09 100644 --- a/apps/cloud/src/account/org-api-key-revoke.node.test.ts +++ b/apps/cloud/src/account/org-api-key-revoke.node.test.ts @@ -8,6 +8,7 @@ import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; import { ORG_SELECTOR_HEADER } from "../auth/organization"; import { WorkOSClient, type WorkOSClientService } from "../auth/workos"; +import { WorkOsMirror } from "../auth/workos-mirror"; import { AutumnService } from "../extensions/billing/service"; import { AccountCaller, workosAccountProvider } from "./workos-account-service"; @@ -100,18 +101,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: org.id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganization: async (id: string) => ({ id, name: `Org ${id}`, slug: id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganizationBySlug: async (slug: string) => ({ id: slug, name: `Org ${slug}`, slug, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), deleteOrganizationCascade: async () => {}, @@ -119,6 +129,22 @@ const stubUsers = Layer.succeed(UserStoreService)({ ), }); +// Revoke changes no membership, so the mirror is never written. +const stubMirror = Layer.succeed(WorkOsMirror)({ + upsertUser: () => Effect.die("revoke does not write the membership mirror"), + upsertMembership: () => Effect.die("revoke does not write the membership mirror"), + deleteMembership: () => Effect.die("revoke does not write the membership mirror"), + deleteUser: () => Effect.die("revoke does not write the membership mirror"), + getCursor: () => Effect.die("revoke does not read the events cursor"), + setCursor: () => Effect.die("revoke does not move the events cursor"), + applyOrganizationScan: () => Effect.die("revoke does not run the backfill"), + replayBoundary: () => Effect.die("revoke does not run the reconciler"), + setReplayBoundary: () => Effect.die("revoke does not run the backfill"), + backfillCompletedAt: () => Effect.die("revoke does not check mirror readiness"), + markBackfillCompleted: () => Effect.die("revoke does not run the backfill"), + organizationBackfilledAt: () => Effect.die("revoke does not report seats"), +}); + const stubAutumn = Layer.succeed(AutumnService)({ use: () => Effect.die("revoke does not touch billing"), ensureCustomer: () => Effect.die("revoke does not touch billing"), @@ -156,6 +182,7 @@ const providerWith = (accountId: string) => { Layer.mergeAll( stubWorkOS, stubUsers, + stubMirror, stubApiKeys, stubAutumn, Layer.succeed(AccountCaller)({ session: session(accountId) }), diff --git a/apps/cloud/src/account/workos-account-service.ts b/apps/cloud/src/account/workos-account-service.ts index dc8b234b1f..beb9ef86a3 100644 --- a/apps/cloud/src/account/workos-account-service.ts +++ b/apps/cloud/src/account/workos-account-service.ts @@ -12,6 +12,7 @@ import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; import type { Session } from "../auth/middleware"; import { WorkOSClient } from "../auth/workos"; +import { WorkOsMirror, mirrorMembershipFromWorkOs } from "../auth/workos-mirror"; import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../auth/organization"; import { AutumnService } from "../extensions/billing/service"; import { forkReportMemberSeats } from "../extensions/billing/member-seats"; @@ -50,7 +51,7 @@ export class AccountCaller extends Context.Service< // (me / API keys) and `org/handlers.ts` (members / roles / invite / role / // name). Native WorkOS / store failures are mapped at this boundary onto the // neutral account errors so the shared UI sees one shape: -// WorkOSError | UserStoreError | ApiKeyManagementError → AccountError +// WorkOSError | UserStoreError | ApiKeyManagementError | WorkOsMirrorError → AccountError // no organization in session → AccountNoOrganization // not-an-admin / over-seat-limit / not-allowed → AccountForbidden // --------------------------------------------------------------------------- @@ -65,13 +66,17 @@ const toAccountError = () => Effect.fail(new AccountError({ message: "Account re export const workosAccountProvider: Layer.Layer< AccountProvider, never, - WorkOSClient | UserStoreService | ApiKeyService | AutumnService | AccountCaller + WorkOSClient | UserStoreService | WorkOsMirror | ApiKeyService | AutumnService | AccountCaller > = Layer.effect(AccountProvider)( Effect.gen(function* () { const workos = yield* WorkOSClient; const apiKeys = yield* ApiKeyService; const autumn = yield* AutumnService; const users = yield* UserStoreService; + // Membership writes below go to WorkOS FIRST (the authority), then are + // written through to the local mirror so the member list and the seat + // count read the change without waiting for the Events reconciler. + const mirror = yield* WorkOsMirror; // The caller, resolved once per request by the cookie-only session // middleware (account-api.ts) — the same credential `SessionAuthLive` @@ -137,6 +142,7 @@ export const workosAccountProvider: Layer.Layer< if (!membership || membership.organizationId !== organizationId) { return yield* new AccountForbidden(); } + return membership; }); // Mirror of org/handlers `getMemberSeats` — live seat usage from WorkOS. @@ -215,7 +221,10 @@ export const workosAccountProvider: Layer.Layer< Effect.gen(function* () { const { session, org } = yield* requireOrganization(headers); const keys = yield* apiKeys - .listUserKeys({ accountId: session.accountId, organizationId: org.id }) + .listUserKeys({ + accountId: session.accountId, + organizationId: org.id, + }) .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError)); return { apiKeys: keys }; }), @@ -225,10 +234,16 @@ export const workosAccountProvider: Layer.Layer< const { session, org } = yield* requireOrganization(headers); const trimmed = name.trim().slice(0, MAX_API_KEY_NAME_LENGTH); if (!trimmed) { - return yield* new AccountError({ message: "API key name is required" }); + return yield* new AccountError({ + message: "API key name is required", + }); } return yield* apiKeys - .createUserKey({ accountId: session.accountId, organizationId: org.id, name: trimmed }) + .createUserKey({ + accountId: session.accountId, + organizationId: org.id, + name: trimmed, + }) .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError)); }), @@ -236,7 +251,10 @@ export const workosAccountProvider: Layer.Layer< Effect.gen(function* () { const { session, org } = yield* requireOrganization(headers); const ownedKeys = yield* apiKeys - .listUserKeys({ accountId: session.accountId, organizationId: org.id }) + .listUserKeys({ + accountId: session.accountId, + organizationId: org.id, + }) .pipe(Effect.catchTag("ApiKeyManagementError", toAccountError)); if (!ownedKeys.some((key) => key.id === apiKeyId)) { return yield* new AccountError({ message: "API key not found" }); @@ -266,7 +284,9 @@ export const workosAccountProvider: Layer.Layer< yield* requireAdmin(session.accountId, org.id); const trimmed = name.trim().slice(0, MAX_API_KEY_NAME_LENGTH); if (!trimmed) { - return yield* new AccountError({ message: "API key name is required" }); + return yield* new AccountError({ + message: "API key name is required", + }); } return yield* apiKeys .createOrgKey({ organizationId: org.id, name: trimmed }) @@ -287,7 +307,11 @@ export const workosAccountProvider: Layer.Layer< yield* apiKeys.revokeOrgKey({ organizationId: org.id, keyId: apiKeyId }).pipe( Effect.catchTag("ApiKeyManagementError", toAccountError), Effect.catchTag("OrgApiKeyNotFound", () => - Effect.fail(new AccountError({ message: "Organization API key not found" })), + Effect.fail( + new AccountError({ + message: "Organization API key not found", + }), + ), ), ); return { success: true }; @@ -361,10 +385,29 @@ export const workosAccountProvider: Layer.Layer< Effect.gen(function* () { const { session, org } = yield* requireOrganization(headers); yield* requireAdmin(session.accountId, org.id); - yield* assertMembershipInOrg(org.id, membershipId); + const membership = yield* assertMembershipInOrg(org.id, membershipId); yield* workos .deleteOrgMembership(membershipId) .pipe(Effect.catchTag("WorkOSError", toAccountError)); + // Tombstoned by identity: the deleted WorkOS id never returns, so + // a login or backfill that fetched this membership before the + // delete — or a role change issued before it and delivered after + // — is refused however it is stamped, while a replacement + // membership WorkOS creates for the same member (a new id) is + // not. No WorkOS instant is in hand (WorkOS answers a delete with + // no time): the row keeps its own stamp, never the local clock, + // which read after WorkOS answered could post-date that + // replacement. + yield* mirror + .deleteMembership( + { + id: membershipId, + accountId: membership.userId, + organizationId: membership.organizationId, + }, + new Date(membership.updatedAt), + ) + .pipe(Effect.catchTag("WorkOsMirrorError", toAccountError)); yield* forkReportMemberSeats(org.id).pipe(Effect.provideContext(ctx)); return { success: true }; }), @@ -374,9 +417,12 @@ export const workosAccountProvider: Layer.Layer< const { session, org } = yield* requireOrganization(headers); yield* requireAdmin(session.accountId, org.id); yield* assertMembershipInOrg(org.id, membershipId); - yield* workos + const updated = yield* workos .updateOrgMembershipRole(membershipId, roleSlug) .pipe(Effect.catchTag("WorkOSError", toAccountError)); + yield* mirror + .upsertMembership(mirrorMembershipFromWorkOs(updated)) + .pipe(Effect.catchTag("WorkOsMirrorError", toAccountError)); return { success: true }; }), @@ -389,7 +435,11 @@ export const workosAccountProvider: Layer.Layer< .pipe(Effect.catchTag("WorkOSError", toAccountError)); yield* users .use("upsertOrganization", (s) => - s.upsertOrganization({ id: updated.id, name: updated.name }), + s.upsertOrganization({ + id: updated.id, + name: updated.name, + updatedAt: new Date(updated.updatedAt), + }), ) .pipe(Effect.catchTag("UserStoreError", toAccountError)); return { name: updated.name }; diff --git a/apps/cloud/src/api/layers.ts b/apps/cloud/src/api/layers.ts index 039d983e55..bcc5c5221d 100644 --- a/apps/cloud/src/api/layers.ts +++ b/apps/cloud/src/api/layers.ts @@ -56,7 +56,9 @@ export const BootSharedServices = Layer.mergeAll( // `AutumnService.Default` is provided HERE because the `createOrganization` // handler reads it for the free-organizations-per-user limit gate — one of the // few app-only billing touchpoints. (It is NOT on the neutral boot core.) -export const makeNonProtectedApiLive = (rsLive: Layer.Layer) => +export const makeNonProtectedApiLive = ( + rsLive: Layer.Layer, +) => HttpApiBuilder.layer(NonProtectedApi).pipe( Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)), Layer.provide(requestScopedMiddleware(rsLive).layer), diff --git a/apps/cloud/src/api/protected-api-key-auth.node.test.ts b/apps/cloud/src/api/protected-api-key-auth.node.test.ts index e87512d285..7be135a9e4 100644 --- a/apps/cloud/src/api/protected-api-key-auth.node.test.ts +++ b/apps/cloud/src/api/protected-api-key-auth.node.test.ts @@ -68,18 +68,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: `org-slug-${org.id}`, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganization: async (id: string) => ({ id, name: `Org ${id}`, slug: `org-slug-${id}`, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganizationBySlug: async (slug: string) => ({ id: "org_by_slug", name: `Org ${slug}`, slug, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), deleteOrganizationCascade: async () => {}, diff --git a/apps/cloud/src/api/protected-jwt-auth.node.test.ts b/apps/cloud/src/api/protected-jwt-auth.node.test.ts index b374810372..e4e2c041d5 100644 --- a/apps/cloud/src/api/protected-jwt-auth.node.test.ts +++ b/apps/cloud/src/api/protected-jwt-auth.node.test.ts @@ -85,18 +85,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: `org-slug-${org.id}`, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganization: async (id: string) => ({ id, name: `Org ${id}`, slug: `org-slug-${id}`, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganizationBySlug: async (slug: string) => ({ id: "org_by_slug", name: `Org ${slug}`, slug, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), deleteOrganizationCascade: async () => {}, diff --git a/apps/cloud/src/api/router.ts b/apps/cloud/src/api/router.ts index d74f9c0b25..227dc20b22 100644 --- a/apps/cloud/src/api/router.ts +++ b/apps/cloud/src/api/router.ts @@ -4,6 +4,7 @@ import { HttpRouter } from "effect/unstable/http"; import { RouterConfigLive, requestScopedMiddleware } from "@executor-js/api/server"; import { UserStoreService } from "../auth/context"; +import { WorkOsMirror } from "../auth/workos-mirror"; import { DbService } from "../db/db"; import { makeAccountApiLive } from "../account/account-api"; @@ -29,7 +30,9 @@ import { makeProtectedApiLive } from "./protected"; // so tests can substitute a counting fake for `DbService.Live` and // assert per-request semantics — see // `apps/cloud/src/api.request-scope.node.test.ts`. -export const makeApiLive = (requestScopedLive: Layer.Layer) => { +export const makeApiLive = ( + requestScopedLive: Layer.Layer, +) => { const BillingRoutesLive = AutumnRoutesLive.pipe( Layer.provide(requestScopedMiddleware(requestScopedLive).layer), ); diff --git a/apps/cloud/src/auth/api.ts b/apps/cloud/src/auth/api.ts index a96a1e21e7..3f69c0ce6c 100644 --- a/apps/cloud/src/auth/api.ts +++ b/apps/cloud/src/auth/api.ts @@ -1,6 +1,6 @@ import { HttpApiEndpoint, HttpApiGroup } from "effect/unstable/httpapi"; import { Schema } from "effect"; -import { UserStoreError, WorkOSError } from "./errors"; +import { UserStoreError, WorkOSError, WorkOsMirrorError } from "./errors"; import { NoOrganization } from "@executor-js/api/server"; import { SessionAuth } from "./middleware"; @@ -172,7 +172,9 @@ export const AUTH_PATHS = { callback: "/api/auth/callback", } as const; -const AuthErrors = [UserStoreError, WorkOSError] as const; +// The login callback and the org handlers feed the membership mirror, so a +// mirror write failure is one of their wire errors (same 500 as a store failure). +const AuthErrors = [UserStoreError, WorkOSError, WorkOsMirrorError] as const; const McpApprovalErrors = [ NoOrganization, McpExecutionNotFoundError, diff --git a/apps/cloud/src/auth/errors.ts b/apps/cloud/src/auth/errors.ts index debf0b0635..6c3c8e3ac2 100644 --- a/apps/cloud/src/auth/errors.ts +++ b/apps/cloud/src/auth/errors.ts @@ -38,6 +38,27 @@ export class UserStoreError extends Schema.TaggedErrorClass()( } } +/** + * The public failure of every cloud membership-mirror write (`WorkOsMirror`). + * Same two diagnosable fields as `UserStoreError` — which mirror call failed, + * and how — classified from the driver cause the same way. Declared here, + * beside `UserStoreError`, because the auth API (`auth/api.ts`, in the SPA + * bundle) names it on the wire for the login and org handlers that feed the + * mirror; the service itself lives in `workos-mirror.ts`. + */ +export class WorkOsMirrorError extends Schema.TaggedErrorClass()( + "WorkOsMirrorError", + { + operation: Schema.String, + reason: Schema.Literals(USER_STORE_FAILURE_REASONS), + }, + { httpApiStatus: 500 }, +) { + override get message(): string { + return `workos mirror ${this.operation} failed: ${this.reason}`; + } +} + /** Reasons a retry can plausibly clear: the query never reached a healthy * server. A `query` failure is deterministic and must not be retried. */ export const isTransientUserStoreReason = (reason: UserStoreFailureReason): boolean => diff --git a/apps/cloud/src/auth/handlers.ts b/apps/cloud/src/auth/handlers.ts index 45dab43006..e7184f253a 100644 --- a/apps/cloud/src/auth/handlers.ts +++ b/apps/cloud/src/auth/handlers.ts @@ -1,6 +1,6 @@ import { HttpApi, HttpApiBuilder } from "effect/unstable/httpapi"; import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; -import { Duration, Effect, Predicate } from "effect"; +import { Clock, Duration, Effect, Predicate } from "effect"; import { isValidOrgSlug } from "@executor-js/api"; import { @@ -18,6 +18,7 @@ import { SessionContext, SessionCookies } from "./middleware"; import { encodeLoginState, decodeLoginState } from "./login-state"; import { safeReturnTo } from "./return-to"; import { UserStoreService } from "./context"; +import { mirrorMembership, mirrorSignIn } from "./mirror-feeders"; import { env } from "cloudflare:workers"; import { WorkOSError } from "./errors"; import { WorkOSClient } from "./workos"; @@ -209,8 +210,16 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( const result = yield* workos.authenticateWithCode(query.code); - // Mirror the account locally - yield* users.use("ensureAccount", (s) => s.ensureAccount(result.user.id)); + // ONE membership list for the whole callback. It feeds the mirror + // (the user + every org they hold a membership in, all already in + // hand) and it is the membership check for every landing-org + // candidate below, so the callback makes no per-candidate WorkOS + // call. The user's account row is minted by the mirror's user + // upsert. The list's fetch instant, taken before the read, stamps + // the organization names it carries (see `mirrorSignIn`). + const fetchedAt = new Date(yield* Clock.currentTimeMillis); + const memberships = yield* workos.listUserMemberships(result.user.id); + yield* mirrorSignIn(result.user, memberships.data, fetchedAt); let sealedSession = result.sealedSession; @@ -220,34 +229,43 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( // any other untrusted path. const returnTo = safeReturnTo(decodeLoginState(query.state)?.returnTo) ?? "/"; const requestedOrgSelector = requestedOrgSelectorFromReturnTo(returnTo); - const requestedOrg = requestedOrgSelector - ? yield* authorizeOrganizationSelector(result.user.id, requestedOrgSelector).pipe( + + // An org SLUG (both candidate sources below are slug-validated, so + // an `org_…` id never reaches here) resolves to its id only when the + // list above holds an ACTIVE membership in it. Pending memberships + // are skipped because refreshing into one 400s and would bypass + // invite consent. A slug that fails to resolve (unknown, or a store + // hiccup) is not a candidate, the same as an org the user is not in. + const activeOrganizationIds = new Set( + memberships.data.filter((m) => m.status === "active").map((m) => m.organizationId), + ); + const activeOrganizationFor = (slug: string) => + users + .use("getOrganizationBySlug", (s) => s.getOrganizationBySlug(slug)) + .pipe( + Effect.map((org) => (org && activeOrganizationIds.has(org.id) ? org.id : null)), Effect.orElseSucceed(() => null), - ) - : null; + ); // Prefer the org in the URL that sent the user to login. If the URL // is bare, or not an org route, prefer the org this browser last // worked in (the last-org cookie — it outlives the session precisely // so a fresh login lands where the user left off), then WorkOS's // org, then the first active membership for org-less sessions. - // Pending memberships are skipped because refreshing into one 400s - // and would bypass invite consent. The cookie is membership-checked - // like any selector, so a stale one just falls through. - let targetOrganizationId = requestedOrg?.id ?? null; + // The cookie is membership-checked like any selector, so a stale + // one just falls through. + let targetOrganizationId = requestedOrgSelector + ? yield* activeOrganizationFor(requestedOrgSelector) + : null; if (!targetOrganizationId && !requestedOrgSelector) { const lastOrgSlug = request.cookies[LAST_ORG_COOKIE]; - const lastOrg = + targetOrganizationId = lastOrgSlug && isValidOrgSlug(lastOrgSlug) - ? yield* authorizeOrganizationSelector(result.user.id, lastOrgSlug).pipe( - Effect.orElseSucceed(() => null), - ) + ? yield* activeOrganizationFor(lastOrgSlug) : null; - targetOrganizationId = lastOrg?.id ?? null; } targetOrganizationId ??= result.organizationId ?? null; if (!targetOrganizationId && !requestedOrgSelector) { - const memberships = yield* workos.listUserMemberships(result.user.id); const existingActive = memberships.data.find((m) => m.status === "active"); targetOrganizationId = existingActive?.organizationId ?? null; } @@ -311,7 +329,9 @@ export const CloudAuthPublicHandlers = HttpApiBuilder.group( ? yield* workos.logoutUrl(sealedSession, origin ? `${origin}/` : undefined) : null; - const response = HttpServerResponse.redirect(logoutUrl ?? "/", { status: 302 }); + const response = HttpServerResponse.redirect(logoutUrl ?? "/", { + status: 302, + }); // Drop only what this browser actually presented. Both cookies are // SameSite=Lax, so a cross-site form POST carries neither — it gets @@ -447,11 +467,18 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( } const org = yield* workos.createOrganization(name); - yield* workos.createMembership(org.id, session.accountId, "admin"); + const membership = yield* workos.createMembership(org.id, session.accountId, "admin"); // `upsertOrganization` mints the slug at insert — no separate heal step. const mirrored = yield* users.use("upsertOrganization", (s) => - s.upsertOrganization({ id: org.id, name: org.name }), + s.upsertOrganization({ + id: org.id, + name: org.name, + updatedAt: new Date(org.updatedAt), + }), ); + // Write-through: the creator's admin membership, from the create + // response, lands in the mirror before anything reads it. + yield* mirrorMembership(membership); // Provision the org's billing customer while we're the ones creating // the org. Without this the first billing call an org ever makes is a @@ -535,13 +562,18 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( // empty workspace when a later request re-mirrors it with a new slug. yield* workos.deleteOrganization(organizationId); - // Purge all local tenant data, secrets, and the identity mirror - // (cascades local memberships) in one transaction. If this fails - // after the WorkOS delete already succeeded, the org is gone for - // everyone (unreachable) but its secrets/tenant rows linger orphaned — - // alert loudly so that window gets swept, then surface the failure. + // Purge all local tenant data, secrets, and the org's memberships in + // one transaction, leaving the org row as a tombstone marked deleted + // (so a login that fetched its membership list before the deletion + // cannot re-mint the org afterwards). If this fails after the WorkOS + // delete already succeeded, the org is gone for everyone + // (unreachable) but its secrets/tenant rows linger orphaned — alert + // loudly so that window gets swept, then surface the failure. + const deletedAt = new Date(yield* Clock.currentTimeMillis); yield* users - .use("deleteOrganizationCascade", (s) => s.deleteOrganizationCascade(organizationId)) + .use("deleteOrganizationCascade", (s) => + s.deleteOrganizationCascade(organizationId, deletedAt), + ) .pipe( Effect.tapError((error) => Effect.logError( @@ -645,9 +677,31 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( // upsert mints the slug at insert — no separate heal step. const org = yield* workos.getOrganization(invitation.organizationId); const mirrored = yield* users.use("upsertOrganization", (s) => - s.upsertOrganization({ id: org.id, name: org.name }), + s.upsertOrganization({ + id: org.id, + name: org.name, + updatedAt: new Date(org.updatedAt), + }), ); + // Write-through: acceptance returns the invitation, not the + // membership it activated, so this is the one feeder that reads the + // membership back (a rare path; one extra call). WorkOS activates it + // as part of acceptance, so its absence is worth a warning — the + // Events reconciler will still land it. + const membership = yield* workos.getUserOrgMembership(org.id, session.accountId); + if (membership) { + yield* mirrorMembership(membership); + } else { + yield* Effect.logWarning( + "acceptInvitation: accepted invitation has no membership yet", + { + userId: session.accountId, + organizationId: org.id, + }, + ); + } + // The membership is active in WorkOS from this point even if // attaching the session below fails, so reconcile the org's billed // seat count now. diff --git a/apps/cloud/src/auth/mirror-feeders.node.test.ts b/apps/cloud/src/auth/mirror-feeders.node.test.ts new file mode 100644 index 0000000000..338ce41391 --- /dev/null +++ b/apps/cloud/src/auth/mirror-feeders.node.test.ts @@ -0,0 +1,1040 @@ +// --------------------------------------------------------------------------- +// The membership mirror's FEEDERS, end to end through the code that runs in +// production, against the real PGlite Postgres every cloud unit test runs on +// (scripts/test-globalsetup.ts). WorkOS is a fake `WorkOSClient` (the +// emulator has no list-users / events routes); the mirror, the user store, +// and the directory read are the live layers over `DbService.Live`. +// +// What this pins: +// - the login callback records the signed-in user and EVERY membership +// WorkOS lists (active and pending), with the org row minted so the FK +// holds — from the one membership list it already fetches +// - the callback picks the landing org from that same list: a returnTo +// slug or last-org cookie lands only in an ACTIVE membership, an unknown +// or pending one falls through +// - `removeMember` tombstones the mirror row after the WorkOS delete, +// stamped with the membership's last WorkOS state (never a local clock), +// so a replay of the membership as it was before the delete cannot +// restore it while a replacement WorkOS created meanwhile is accepted +// - `updateMemberRole` writes the role WorkOS returned +// - the backfill mirrors every org's members and counts what it wrote, +// writes nothing on a dry run, converges on a re-run, tombstones a +// membership WorkOS no longer lists — but never one written after its +// listing was taken — marks each org backfilled as of its listing, and +// records the events replay boundary only when it completes and only +// ONCE: a run that fails part-way keeps the marks of the orgs it +// finished and records no boundary, and a later completed run keeps the +// first boundary (the org renames and user deletions between two runs +// are the events stream's to replay) +// - two scans of one org that overlap cannot resurrect a membership: a scan +// that listed it, stalled, and resumed after a later listing (which no +// longer had it) was applied is refused whole +// - a login whose membership list was fetched BEFORE the org was purged and +// written after cannot re-mint the org or its membership, one fetched +// before a rename cannot revert the rename, and one fetched before a +// revocation the backfill has since scanned cannot reinstate the +// membership +// --------------------------------------------------------------------------- + +import { describe, expect, it } from "@effect/vitest"; +import { sql } from "drizzle-orm"; +import { Effect, Exit, Fiber, Latch, Layer } from "effect"; +import { HttpRouter, HttpServer } from "effect/unstable/http"; +import { HttpApiBuilder } from "effect/unstable/httpapi"; + +import { + AccountProvider, + MemberDirectory, + RouterConfigLive, + requestScopedMiddleware, +} from "@executor-js/api/server"; + +import { AccountCaller, workosAccountProvider } from "../account/workos-account-service"; +import { RequestScopedServicesLive } from "../api/layers"; +import { DbService } from "../db/db"; +import { AutumnService } from "../extensions/billing/service"; +import { ApiKeyService } from "./api-keys"; +import { UserStoreService } from "./context"; +import { WorkOSError } from "./errors"; +import { CloudAuthPublicHandlers, CloudSessionAuthHandlers, NonProtectedApi } from "./handlers"; +import { LAST_ORG_COOKIE } from "./last-org-cookie"; +import { encodeLoginState } from "./login-state"; +import { cloudMemberDirectoryLayer } from "./member-directory"; +import { SessionAuthLive } from "./middleware-live"; +import { mirrorSignIn } from "./mirror-feeders"; +import { ORG_SELECTOR_HEADER } from "./organization"; +import { WorkOSClient, type WorkOSClientService } from "./workos"; +import { WorkOsMirror, type WorkOsMirrorShape } from "./workos-mirror"; +import { backfillOrganization, backfillWorkOsMirror } from "./workos-mirror-backfill"; +import type { WorkOsMembershipPayload, WorkOsUserPayload } from "./workos-mirror-store"; + +const T1 = "2026-01-01T00:00:00.000Z"; +const T2 = "2026-01-02T00:00:00.000Z"; + +// Synthetic identities only. Every test mints its own org ids so the shared +// test database never couples two tests. +const freshId = (prefix: string) => `${prefix}_${crypto.randomUUID().replaceAll("-", "")}`; + +const workosUser = (id: string, overrides: Partial = {}) => ({ + object: "user" as const, + id, + email: `${id}@placeholder.test`, + emailVerified: true, + firstName: "Ada", + lastName: "Placeholder", + profilePictureUrl: null, + lastSignInAt: T1, + locale: null, + createdAt: T1, + updatedAt: T1, + externalId: null, + metadata: {}, + ...overrides, +}); + +interface FakeMembership extends WorkOsMembershipPayload { + readonly organizationName: string; +} + +const workosMembership = ( + userId: string, + organizationId: string, + overrides: Partial = {}, +): FakeMembership => ({ + id: `om_${userId}_${organizationId}`, + userId, + organizationId, + organizationName: `Org ${organizationId}`, + role: { slug: "member" }, + status: "active", + updatedAt: T1, + ...overrides, +}); + +/** Mirrored rows for one org, read through the live cloud `MemberDirectory`. */ +const readMembers = (organizationId: string) => + Effect.runPromise( + Effect.flatMap(MemberDirectory.asEffect(), (directory) => + directory.members(organizationId, { + statuses: ["active", "pending", "inactive"], + }), + ).pipe( + Effect.provide(cloudMemberDirectoryLayer.pipe(Layer.provide(DbService.Live))), + Effect.scoped, + ), + ); + +/** Mirror an org row (named as of T1) and return the URL slug the store minted for it. */ +const seedOrganization = (id: string) => + Effect.runPromise( + Effect.flatMap(UserStoreService.asEffect(), (users) => + users.use("upsertOrganization", (s) => + s.upsertOrganization({ + id, + name: `Org ${id}`, + updatedAt: new Date(T1), + }), + ), + ).pipe( + Effect.map((org) => org.slug), + Effect.provide(UserStoreService.Live.pipe(Layer.provide(DbService.Live))), + Effect.scoped, + ), + ); + +const stubAutumn = Layer.succeed(AutumnService)({ + use: () => Effect.die("feeders do not read billing"), + ensureCustomer: () => Effect.void, + checkExecutionBalance: () => Effect.die("feeders do not check balances"), + trackExecution: () => Effect.void, + setMemberSeats: () => Effect.void, +}); + +/** + * A `WorkOSClient` whose every method is one of `methods`; anything else is + * an unexpected call and dies, so a feeder that silently adds a WorkOS read + * fails the test instead of passing on a fake. + */ +const stubWorkOS = (methods: Partial) => + Layer.succeed( + WorkOSClient, + new Proxy({} as WorkOSClientService, { + get: (_target, prop) => + (methods as Record)[prop] ?? + (() => Effect.die(`unexpected WorkOSClient.${String(prop)} call`)), + }), + ); + +describe("login callback", () => { + const callbackHandler = (workos: Layer.Layer) => + HttpRouter.toWebHandler( + HttpApiBuilder.layer(NonProtectedApi).pipe( + Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)), + Layer.provide(requestScopedMiddleware(RequestScopedServicesLive).layer), + Layer.provideMerge(SessionAuthLive), + Layer.provideMerge(stubAutumn), + Layer.provideMerge(workos), + Layer.provideMerge(HttpServer.layerServices), + Layer.provideMerge(RouterConfigLive), + ), + { disableLogger: true }, + ).handler; + + const STATE_COOKIE = "wos-login-state"; + + /** + * A callback handler over a fake WorkOS that authenticates `user` with the + * memberships `listed`, recording every WorkOS read (`calls`) and every + * session refresh (`refreshedInto`, the org ids) so the landing-org choice + * is assertable from the outside. + */ + const signIn = (user: ReturnType, listed: readonly FakeMembership[]) => { + const calls: string[] = []; + const refreshedInto: (string | undefined)[] = []; + const handler = callbackHandler( + stubWorkOS({ + authenticateWithCode: () => + Effect.succeed({ + user, + organizationId: undefined, + accessToken: "access", + refreshToken: "refresh", + sealedSession: "sealed", + }), + listUserMemberships: (id) => { + calls.push(`listUserMemberships:${id}`); + return Effect.succeed({ + object: "list" as const, + data: listed as never[], + listMetadata: { before: null, after: null }, + }); + }, + // The forked seat recount after login. + listOrgMembers: () => + Effect.succeed({ + object: "list" as const, + data: [] as never[], + listMetadata: { before: null, after: null }, + }), + refreshSession: (_sealed, organizationId) => { + refreshedInto.push(organizationId); + return Effect.succeed("sealed-refreshed"); + }, + }), + ); + return { handler, calls, refreshedInto }; + }; + + /** + * `GET /auth/callback` with the CSRF-matched login `state` (the callback + * refuses any request without one) and any extra cookies; `returnTo` + * rides inside the state as /login mints it. + */ + const callbackRequest = (options: { returnTo?: string; cookies?: Record }) => { + const url = new URL("http://test.local/auth/callback"); + url.searchParams.set("code", "code_1"); + const state = encodeLoginState({ + nonce: "nonce", + ...(options.returnTo === undefined ? {} : { returnTo: options.returnTo }), + }); + url.searchParams.set("state", state); + const cookies = { ...options.cookies, [STATE_COOKIE]: state }; + const cookie = Object.entries(cookies) + .map(([name, value]) => `${name}=${value}`) + .join("; "); + return new Request(url, { headers: cookie ? { cookie } : {} }); + }; + + it("records the user and every listed membership from the one list it already fetches", async () => { + const userId = freshId("user"); + const activeOrg = freshId("org"); + const pendingOrg = freshId("org"); + const { handler, calls } = signIn( + workosUser(userId, { + firstName: "Grace", + lastName: "Hopper", + updatedAt: T2, + }), + [ + workosMembership(userId, activeOrg, { + role: { slug: "admin" }, + updatedAt: T2, + }), + workosMembership(userId, pendingOrg, { status: "pending" }), + ], + ); + + const response = await handler(callbackRequest({})); + + expect(response.status).toBe(302); + expect(calls, "one membership list for the whole callback").toEqual([ + `listUserMemberships:${userId}`, + ]); + + const active = await readMembers(activeOrg); + expect(active).toHaveLength(1); + expect(active[0]).toMatchObject({ + accountId: userId, + membershipId: `om_${userId}_${activeOrg}`, + email: `${userId}@placeholder.test`, + name: "Grace Hopper", + role: "admin", + status: "active", + lastActiveAt: new Date(T1).getTime(), + }); + const pending = await readMembers(pendingOrg); + expect( + pending.map((m) => m.status), + "pending memberships are mirrored too", + ).toEqual(["pending"]); + }); + + describe("lands in the org the returnTo slug names", () => { + it("when the user holds an active membership there", async () => { + const userId = freshId("user"); + const requested = freshId("org"); + const other = freshId("org"); + const slug = await seedOrganization(requested); + const { handler, refreshedInto } = signIn(workosUser(userId), [ + workosMembership(userId, other), + workosMembership(userId, requested), + ]); + + const response = await handler(callbackRequest({ returnTo: `/${slug}/settings` })); + + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe(`/${slug}/settings`); + expect(refreshedInto, "the session is switched into the requested org").toEqual([requested]); + }); + + it("never when the membership there is only pending", async () => { + const userId = freshId("user"); + const requested = freshId("org"); + const other = freshId("org"); + const slug = await seedOrganization(requested); + const { handler, refreshedInto } = signIn(workosUser(userId), [ + workosMembership(userId, other), + workosMembership(userId, requested, { status: "pending" }), + ]); + + const response = await handler(callbackRequest({ returnTo: `/${slug}` })); + + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe(`/${slug}`); + expect( + refreshedInto, + "a pending membership is not a landing candidate, and an explicit slug does not fall back to another org", + ).toEqual([]); + }); + }); + + describe("without a returnTo org", () => { + it("lands in the last-org cookie's org when the user is active there", async () => { + const userId = freshId("user"); + const last = freshId("org"); + const other = freshId("org"); + const slug = await seedOrganization(last); + const { handler, refreshedInto } = signIn(workosUser(userId), [ + workosMembership(userId, other), + workosMembership(userId, last), + ]); + + const response = await handler(callbackRequest({ cookies: { [LAST_ORG_COOKIE]: slug } })); + + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe("/"); + expect(refreshedInto).toEqual([last]); + }); + + it("falls through an unknown last-org slug to the first active membership", async () => { + const userId = freshId("user"); + const pendingOrg = freshId("org"); + const activeOrg = freshId("org"); + const { handler, refreshedInto } = signIn(workosUser(userId), [ + workosMembership(userId, pendingOrg, { status: "pending" }), + workosMembership(userId, activeOrg), + ]); + + const response = await handler( + // Valid slug grammar, never minted: the store finds no org for it. + callbackRequest({ cookies: { [LAST_ORG_COOKIE]: "no-such-org-slug" } }), + ); + + expect(response.status).toBe(302); + expect(refreshedInto).toEqual([activeOrg]); + }); + }); +}); + +describe("a delayed sign-in feeder", () => { + /** The live mirror, user store, and directory over one test-db socket. */ + const Services = Layer.mergeAll( + UserStoreService.Live, + WorkOsMirror.Live, + cloudMemberDirectoryLayer, + ).pipe(Layer.provideMerge(DbService.Live)); + + const run = ( + body: Effect.Effect, + ) => Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped)); + + const readOrganization = (org: string) => + Effect.flatMap(UserStoreService.asEffect(), (users) => + users.use("getOrganization", (s) => s.getOrganization(org)), + ); + + const readMembership = (userId: string, org: string) => + Effect.flatMap(MemberDirectory.asEffect(), (directory) => + directory.membership(userId, org, ["active", "pending", "inactive"]), + ); + + it("cannot re-mint a purged organization or its membership from a list fetched before the purge", async () => { + const userId = freshId("user"); + const org = freshId("org"); + await seedOrganization(org); + const result = await run( + Effect.gen(function* () { + const users = yield* UserStoreService; + // The login fetched its membership list at T1, while the org lived... + const fetchedAt = new Date(T1); + const listed = [workosMembership(userId, org)]; + // ...then stalled while cloud's deletion flow purged the org at T2. + yield* users.use("deleteOrganizationCascade", (s) => + s.deleteOrganizationCascade(org, new Date(T2)), + ); + // The stalled login resumes and writes what it holds. + yield* mirrorSignIn(workosUser(userId), listed, fetchedAt); + return { + organization: yield* readOrganization(org), + membership: yield* readMembership(userId, org), + }; + }), + ); + expect(result.organization?.deletedAt, "the org stays a deleted tombstone").toEqual( + new Date(T2), + ); + expect(result.membership, "and holds no membership: nothing to authorize").toBeNull(); + }); + + it("cannot reinstate a membership from a list fetched before a revocation the backfill has since scanned", async () => { + const userId = freshId("user"); + const org = freshId("org"); + await seedOrganization(org); + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + // The login fetched its membership list at T1, while the user was + // a member, then stalled... + const fetchedAt = new Date(T1); + const listed = [workosMembership(userId, org)]; + // ...WorkOS revoked the membership before the mirror was ever + // backfilled, and the backfill then scanned the org at T2 without + // it: no tombstone, the row was never there — and the revocation + // predates the events replay boundary, so no event will land it. + yield* mirror.applyOrganizationScan({ + organizationId: org, + listedAt: new Date(T2), + members: [], + }); + // The stalled login resumes and writes what it holds. + yield* mirrorSignIn(workosUser(userId), listed, fetchedAt); + const membership = yield* readMembership(userId, org); + // A login after the scan, carrying a membership WorkOS created + // since (stamped past the scan), is recorded. + const rejoinedAt = "2026-01-03T00:00:00.000Z"; + yield* mirrorSignIn( + workosUser(userId), + [workosMembership(userId, org, { id: `om_${userId}_${org}_2`, updatedAt: rejoinedAt })], + new Date(rejoinedAt), + ); + return { membership, rejoined: yield* readMembership(userId, org) }; + }), + ); + expect(result.membership, "the pre-scan list reinstates nothing").toBeNull(); + expect(result.rejoined?.membershipId, "a membership newer than the scan is recorded").toBe( + `om_${userId}_${org}_2`, + ); + }); + + it("cannot revert a rename from a list fetched before it, and applies a newer name", async () => { + const userId = freshId("user"); + const org = freshId("org"); + await seedOrganization(org); + const result = await run( + Effect.gen(function* () { + const users = yield* UserStoreService; + // The org is renamed through Executor (write-through of the WorkOS + // organization payload, stamped T2)... + yield* users.use("upsertOrganization", (s) => + s.upsertOrganization({ + id: org, + name: "Renamed Org", + updatedAt: new Date(T2), + }), + ); + // ...after a login had fetched a list still carrying the old name at T1. + yield* mirrorSignIn( + workosUser(userId), + [workosMembership(userId, org, { organizationName: `Org ${org}` })], + new Date(T1), + ); + const afterStale = yield* readOrganization(org); + // A login whose list was fetched after the rename carries the new name. + yield* mirrorSignIn( + workosUser(userId), + [ + workosMembership(userId, org, { + organizationName: "Renamed Again", + }), + ], + new Date("2026-01-03T00:00:00.000Z"), + ); + const afterNewer = yield* readOrganization(org); + return { + afterStale, + afterNewer, + membership: yield* readMembership(userId, org), + }; + }), + ); + expect(result.afterStale?.name, "the stale list does not revert the rename").toBe( + "Renamed Org", + ); + expect(result.afterStale?.slug, "and the slug is untouched").toBe(result.afterNewer?.slug); + expect(result.afterNewer?.name, "a list fetched after the rename is applied").toBe( + "Renamed Again", + ); + expect(result.membership?.status, "the membership itself is recorded either way").toBe( + "active", + ); + }); +}); + +describe("account service writes through to the mirror", () => { + const ADMIN = freshId("user"); + const TARGET = freshId("user"); + + const session = (accountId: string) => ({ + accountId, + email: `${accountId}@placeholder.test`, + name: null, + avatarUrl: null, + organizationId: null, + sealedSession: "sealed", + refreshedSession: null, + }); + + const stubApiKeys = Layer.succeed(ApiKeyService)({ + validate: () => Effect.die("membership writes do not validate keys"), + listUserKeys: () => Effect.die("membership writes do not list keys"), + createUserKey: () => Effect.die("membership writes do not create keys"), + revokeUserKey: () => Effect.die("membership writes do not revoke keys"), + listOrgKeys: () => Effect.die("membership writes do not list keys"), + createOrgKey: () => Effect.die("membership writes do not create keys"), + revokeOrgKey: () => Effect.die("membership writes do not revoke keys"), + }); + + /** + * The provider layer over the LIVE mirror + user store (test db) and a fake + * WorkOS in which ADMIN administers `org` and TARGET is a plain member. + * `deleted` records the WorkOS-side deletes so "WorkOS first" is assertable. + * Provided around the WHOLE test body so the postgres socket outlives the + * provider call under test. + */ + const providerLayer = (org: string, deleted: string[]) => { + const list = (data: readonly unknown[]) => + Effect.succeed({ + object: "list" as const, + data: data as never[], + listMetadata: { before: null, after: null }, + }); + const workos = stubWorkOS({ + listUserMemberships: (userId) => list([workosMembership(userId, org)]), + getUserOrgMembership: (organizationId, userId) => + Effect.succeed( + workosMembership(userId, organizationId, { + role: { slug: userId === ADMIN ? "admin" : "member" }, + }) as never, + ), + getOrgMembership: (membershipId) => + Effect.succeed(workosMembership(TARGET, org, { id: membershipId }) as never), + deleteOrgMembership: (membershipId) => + Effect.sync(() => { + deleted.push(membershipId); + }), + updateOrgMembershipRole: (membershipId, roleSlug) => + Effect.succeed( + workosMembership(TARGET, org, { + id: membershipId, + role: { slug: roleSlug }, + updatedAt: T2, + }) as never, + ), + listOrgMembers: () => list([]), + }); + // The test database serves ONE connection at a time, so the seed, the + // provider, and the directory read all share this layer's socket. + const stores = Layer.mergeAll( + UserStoreService.Live, + WorkOsMirror.Live, + cloudMemberDirectoryLayer, + ); + return workosAccountProvider.pipe( + Layer.provide( + Layer.mergeAll( + workos, + stubApiKeys, + stubAutumn, + Layer.succeed(AccountCaller)({ session: session(ADMIN) }), + ), + ), + Layer.provideMerge(stores), + Layer.provide(DbService.Live), + ); + }; + + // TARGET as an existing member of `org`, seeded through the live mirror. + const seedTarget = (org: string) => + Effect.gen(function* () { + const users = yield* UserStoreService; + const mirror = yield* WorkOsMirror; + yield* users.use("upsertOrganization", (s) => + s.upsertOrganization({ + id: org, + name: `Org ${org}`, + updatedAt: new Date(T1), + }), + ); + yield* mirror.upsertMembership({ + id: `om_${TARGET}_${org}`, + accountId: TARGET, + organizationId: org, + role: "member", + status: "active", + updatedAt: new Date(T1), + }); + }); + + const membersOf = (org: string) => + Effect.flatMap(MemberDirectory.asEffect(), (directory) => directory.members(org)); + + it.effect("removeMember tombstones the mirror row after the WorkOS delete", () => { + const org = freshId("org"); + const deleted: string[] = []; + return Effect.gen(function* () { + yield* seedTarget(org); + const account = yield* AccountProvider; + + const result = yield* account.removeMember( + { [ORG_SELECTOR_HEADER]: org }, + `om_${TARGET}_${org}`, + ); + + expect(result).toEqual({ success: true }); + expect(deleted, "WorkOS is the authority and is written first").toEqual([ + `om_${TARGET}_${org}`, + ]); + const members = yield* membersOf(org); + expect(members.map((m) => m.accountId)).not.toContain(TARGET); + + // A login or backfill that listed TARGET's membership BEFORE the + // removal writes it afterwards: the tombstone refuses it. + const mirror = yield* WorkOsMirror; + const replayed = yield* mirror.upsertMembership({ + id: `om_${TARGET}_${org}`, + accountId: TARGET, + organizationId: org, + role: "member", + status: "active", + updatedAt: new Date(T1), + }); + expect(replayed, "the pre-removal payload is refused").toBe(false); + expect((yield* membersOf(org)).map((m) => m.accountId)).not.toContain(TARGET); + + // The tombstone carries the membership's last WorkOS stamp (T1), not + // the wall clock at the delete: a replacement membership WorkOS + // created for TARGET while the removal was in flight — stamped T2, + // long before any clock this test runs under — is accepted. + const replaced = yield* mirror.upsertMembership({ + id: `om_${TARGET}_${org}_2`, + accountId: TARGET, + organizationId: org, + role: "member", + status: "active", + updatedAt: new Date(T2), + }); + expect(replaced, "a replacement newer than the removed state is accepted").toBe(true); + expect((yield* membersOf(org)).map((m) => m.accountId)).toContain(TARGET); + }).pipe(Effect.provide(providerLayer(org, deleted))); + }); + + it.effect("updateMemberRole writes the role WorkOS returned", () => { + const org = freshId("org"); + return Effect.gen(function* () { + yield* seedTarget(org); + const account = yield* AccountProvider; + + const result = yield* account.updateMemberRole( + { [ORG_SELECTOR_HEADER]: org }, + `om_${TARGET}_${org}`, + "admin", + ); + + expect(result).toEqual({ success: true }); + const members = yield* membersOf(org); + expect(members.find((m) => m.accountId === TARGET)?.role).toBe("admin"); + }).pipe(Effect.provide(providerLayer(org, []))); + }); +}); + +describe("backfill", () => { + /** A fake WorkOS holding `orgs` → members, counting `getUser` calls. */ + const source = (orgs: ReadonlyMap, userCalls: string[]) => ({ + listOrganizationIds: () => Effect.succeed([...orgs.keys()]), + listOrgMembers: (organizationId: string) => Effect.succeed(orgs.get(organizationId) ?? []), + getUser: (userId: string) => + Effect.sync(() => { + userCalls.push(userId); + return workosUser(userId); + }), + }); + + const withMirror = (body: (mirror: WorkOsMirrorShape) => Effect.Effect) => + Effect.runPromise( + Effect.flatMap(WorkOsMirror.asEffect(), body).pipe( + Effect.provide(WorkOsMirror.Live.pipe(Layer.provide(DbService.Live))), + Effect.scoped, + ), + ); + + const runBackfill = ( + orgs: ReadonlyMap, + dryRun: boolean, + userCalls: string[] = [], + ) => + withMirror((mirror) => + backfillWorkOsMirror(source(orgs, userCalls), mirror, { + dryRun, + log: () => undefined, + }), + ); + + /** The instance-wide replay boundary a completed run records. */ + const syncState = () => withMirror((mirror) => mirror.replayBoundary()); + + /** When a run first covered every organization, or null: the authorization gate's first half. */ + const completedAt = () => withMirror((mirror) => mirror.backfillCompletedAt()); + + /** Drop the instance-wide events row, so the run under test is the first ever. */ + const clearEventsRow = () => + Effect.runPromise( + Effect.flatMap(DbService.asEffect(), ({ db }) => + Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)), + ).pipe(Effect.provide(DbService.Live), Effect.scoped), + ); + + const backfilledAt = (org: string) => + withMirror((mirror) => mirror.organizationBackfilledAt(org)); + + it("records the replay boundary on first completion, marks and mirrors every organization's members, counts the writes, converges and repairs on a re-run", async () => { + const orgA = freshId("org"); + const orgB = freshId("org"); + await seedOrganization(orgA); + await seedOrganization(orgB); + const shared = freshId("user"); + const leaving = freshId("user"); + const orgs = new Map([ + [orgA, [workosMembership(shared, orgA), workosMembership(leaving, orgA)]], + [orgB, [workosMembership(shared, orgB, { status: "pending" })]], + ]); + + // The boundary is instance-wide (migration 0019 seeds it on the empty + // test database, other tests may have written it): start as a database + // that has never been backfilled. + await clearEventsRow(); + const startedAt = Date.now(); + + const dry = await runBackfill(orgs, true); + expect(dry).toEqual({ + organizations: 2, + memberships: 3, + usersWritten: 0, + membershipsWritten: 0, + membershipsTombstoned: 0, + }); + expect(await readMembers(orgA), "a dry run writes nothing").toEqual([]); + expect(await syncState(), "a dry run records no boundary").toBeNull(); + expect(await completedAt(), "nor a completion").toBeNull(); + expect(await backfilledAt(orgA), "and marks nothing").toBeNull(); + + const userCalls: string[] = []; + const first = await runBackfill(orgs, false, userCalls); + expect(first).toEqual({ + organizations: 2, + memberships: 3, + usersWritten: 3, + membershipsWritten: 3, + membershipsTombstoned: 0, + }); + expect(userCalls, "one getUser per membership").toHaveLength(3); + expect((await readMembers(orgA)).map((m) => m.status)).toEqual(["active", "active"]); + expect((await readMembers(orgB)).map((m) => m.status)).toEqual(["pending"]); + const after = await syncState(); + expect(after, "the run records where the events replay starts").not.toBeNull(); + const firstCompletion = await completedAt(); + expect(firstCompletion, "and that every organization is now covered").not.toBeNull(); + expect(firstCompletion!.getTime()).toBeGreaterThanOrEqual(after!.getTime()); + expect(after!.getTime()).toBeGreaterThanOrEqual(startedAt); + for (const org of [orgA, orgB]) { + const marked = await backfilledAt(org); + expect(marked, "each scanned organization is marked as of its listing").not.toBeNull(); + expect(marked!.getTime()).toBeGreaterThanOrEqual(after!.getTime()); + } + + // Same payloads again, minus one member WorkOS no longer lists: the + // `updatedAt` guard lets equal payloads through (replays converge), and + // the missing membership is tombstoned — a re-run repairs a stale row. + orgs.set(orgA, [workosMembership(shared, orgA)]); + const again = await runBackfill(orgs, false); + expect(again).toMatchObject({ memberships: 2, membershipsTombstoned: 1 }); + expect( + await syncState(), + "the re-run keeps the first boundary: an org rename or user deletion between the two runs is only in the events stream", + ).toEqual(after); + expect(await completedAt(), "and the first completion").toEqual(firstCompletion); + expect(new Map((await readMembers(orgA)).map((m) => [m.accountId, m.status]))).toEqual( + new Map([ + [shared, "active"], + [leaving, "inactive"], + ]), + ); + expect( + (await readMembers(orgB)).map((m) => m.status), + "the other org is untouched", + ).toEqual(["pending"]); + // The tombstone is keyed to the deleted membership id, so the member's + // pre-removal payload (as a late login or event would carry) is refused. + const replayed = await withMirror((mirror) => + mirror.upsertMembership({ + id: `om_${leaving}_${orgA}`, + accountId: leaving, + organizationId: orgA, + role: "member", + status: "active", + updatedAt: new Date(T1), + }), + ); + expect(replayed).toBe(false); + }); + + it("keeps a membership WorkOS merely deactivated as inactive, not tombstoned, so a later reactivation lands", async () => { + const org = freshId("org"); + await seedOrganization(org); + const paused = freshId("user"); + // Mirrored while active (a sign-in), then deactivated in WorkOS: the + // listing carries the membership under the SAME id with its real status. + await withMirror((mirror) => + mirror.upsertMembership({ + id: `om_${paused}_${org}`, + accountId: paused, + organizationId: org, + role: "member", + status: "active", + updatedAt: new Date(T1), + }), + ); + + const counts = await runBackfill( + new Map([[org, [workosMembership(paused, org, { status: "inactive", updatedAt: T2 })]]]), + false, + ); + expect(counts, "the deactivated membership is written, not tombstoned").toMatchObject({ + memberships: 1, + membershipsWritten: 1, + membershipsTombstoned: 0, + }); + expect((await readMembers(org)).map((m) => [m.accountId, m.status])).toEqual([ + [paused, "inactive"], + ]); + + // WorkOS reactivates it under the same id AFTER the scan (so the payload + // is stamped past the org's `backfilled_at`): an ordinary newer payload, + // which a tombstone keyed to that id would have refused for good. + const reactivated = await withMirror((mirror) => + mirror.upsertMembership({ + id: `om_${paused}_${org}`, + accountId: paused, + organizationId: org, + role: "member", + status: "active", + updatedAt: new Date(Date.now() + 60 * 1000), + }), + ); + expect(reactivated).toBe(true); + expect((await readMembers(org)).map((m) => m.status)).toEqual(["active"]); + }); + + it("leaves a membership written after its listing alone when tombstoning what the listing lacks", async () => { + const org = freshId("org"); + await seedOrganization(org); + const listed = freshId("user"); + const stale = freshId("user"); + const joinedMeanwhile = freshId("user"); + // Both rows are absent from the listing below. `stale` is stamped before + // the listing (a genuine leaver); `joinedMeanwhile` carries a stamp AFTER + // any listing this run can take — the membership WorkOS created between + // the listing and the cleanup, whose own event lands via the reconciler. + const afterListing = new Date(Date.now() + 60 * 60 * 1000); + await withMirror((mirror) => + Effect.all([ + mirror.upsertMembership({ + id: `om_${stale}_${org}`, + accountId: stale, + organizationId: org, + role: "member", + status: "active", + updatedAt: new Date(T1), + }), + mirror.upsertMembership({ + id: `om_${joinedMeanwhile}_${org}`, + accountId: joinedMeanwhile, + organizationId: org, + role: "member", + status: "active", + updatedAt: afterListing, + }), + ]), + ); + + const counts = await runBackfill(new Map([[org, [workosMembership(listed, org)]]]), false); + + expect(counts).toMatchObject({ memberships: 1, membershipsTombstoned: 1 }); + expect(new Map((await readMembers(org)).map((m) => [m.accountId, m.status]))).toEqual( + new Map([ + [listed, "active"], + [stale, "inactive"], + [joinedMeanwhile, "active"], + ]), + ); + }); + + it("keeps the marks of the organizations it finished but records no replay boundary when a run fails part-way", async () => { + const orgA = freshId("org"); + const orgB = freshId("org"); + await seedOrganization(orgA); + await seedOrganization(orgB); + const member = freshId("user"); + const orgs = new Map([ + [orgA, [workosMembership(member, orgA)]], + [orgB, [workosMembership(member, orgB)]], + ]); + // A completed run first, so there IS a boundary to protect. + await runBackfill(orgs, false); + const completed = await syncState(); + expect(completed).not.toBeNull(); + const markedA = await backfilledAt(orgA); + + // A re-run whose second organization fails on a WorkOS read: the first + // org was written, but the run as a whole did not complete — and even a + // completed one would keep the first boundary. + const failing = { + ...source(orgs, []), + listOrgMembers: (organizationId: string) => + organizationId === orgB + ? Effect.fail(new WorkOSError({ status: 503 })) + : Effect.succeed(orgs.get(organizationId) ?? []), + }; + const exit = await withMirror((mirror) => + Effect.exit( + backfillWorkOsMirror(failing, mirror, { + dryRun: false, + log: () => undefined, + }), + ), + ); + expect(Exit.isFailure(exit), "the run fails rather than skipping the org").toBe(true); + expect(await syncState(), "the completed run's boundary stands").toEqual(completed); + expect(await completedAt(), "and so does its completion mark").not.toBeNull(); + expect( + (await backfilledAt(orgA))!.getTime(), + "the org the failed run did finish is marked as of its new listing", + ).toBeGreaterThanOrEqual(markedA!.getTime()); + }); + + it("scans one organization on demand and marks only that one", async () => { + const org = freshId("org"); + const other = freshId("org"); + await seedOrganization(org); + await seedOrganization(other); + const member = freshId("user"); + const orgs = new Map([ + [org, [workosMembership(member, org)]], + [other, [workosMembership(member, other)]], + ]); + + const counts = await withMirror((mirror) => + backfillOrganization(source(orgs, []), mirror, org, { dryRun: false }), + ); + + expect(counts).toEqual({ + applied: true, + memberships: 1, + usersWritten: 1, + membershipsWritten: 1, + membershipsTombstoned: 0, + }); + expect((await readMembers(org)).map((m) => m.accountId)).toEqual([member]); + expect(await backfilledAt(org)).not.toBeNull(); + expect(await readMembers(other), "the other organization is not scanned").toEqual([]); + expect(await backfilledAt(other), "nor marked").toBeNull(); + }); + + it("refuses a scan that stalled while a later scan found a membership gone, so the revoked member stays revoked", async () => { + const org = freshId("org"); + await seedOrganization(org); + const staying = freshId("user"); + const leaving = freshId("user"); + const result = await withMirror((mirror) => + Effect.gen(function* () { + // Scan A lists the org while `leaving` is still a member, then + // stalls (its listing is held behind the latch)... + const listedByA = yield* Latch.make(false); + const stalled = { + ...source(new Map(), []), + listOrgMembers: () => + listedByA.await.pipe( + Effect.as([workosMembership(staying, org), workosMembership(leaving, org)]), + ), + }; + const scanA = yield* Effect.forkChild( + backfillOrganization(stalled, mirror, org, { dryRun: false }), + { startImmediately: true }, + ); + // ...WorkOS removes `leaving`, and scan B lists and applies the + // org without them — no tombstone, the row was never there... + const b = yield* backfillOrganization( + source(new Map([[org, [workosMembership(staying, org)]]]), []), + mirror, + org, + { dryRun: false }, + ); + // ...then A resumes with its older listing. + yield* listedByA.open; + const a = yield* Fiber.join(scanA); + return { a, b }; + }), + ); + expect(result.b).toMatchObject({ applied: true, membershipsWritten: 1 }); + expect(result.a, "the older listing is refused whole").toMatchObject({ + applied: false, + memberships: 2, + usersWritten: 0, + membershipsWritten: 0, + membershipsTombstoned: 0, + }); + expect( + new Map((await readMembers(org)).map((m) => [m.accountId, m.status])), + "the member the later listing no longer had was never inserted", + ).toEqual(new Map([[staying, "active"]])); + }); +}); diff --git a/apps/cloud/src/auth/mirror-feeders.ts b/apps/cloud/src/auth/mirror-feeders.ts new file mode 100644 index 0000000000..0dc825e9b3 --- /dev/null +++ b/apps/cloud/src/auth/mirror-feeders.ts @@ -0,0 +1,80 @@ +// --------------------------------------------------------------------------- +// The membership mirror's request-path FEEDERS: the writes the login callback +// and the Executor-initiated membership changes make through `WorkOsMirror`. +// +// Each feeder takes the WorkOS payload the caller ALREADY holds (the +// authenticated user, the membership list the callback fetches to pick a +// landing org, the membership a write returned) so feeding the mirror never +// adds a WorkOS read. Mirror failures fail the request: the mirror is the +// membership read path, so a login that could not record its memberships is +// not a login that finished. +// --------------------------------------------------------------------------- + +import { Effect } from "effect"; + +import { UserStoreService } from "./context"; +import { + WorkOsMirror, + mirrorMembershipFromWorkOs, + mirrorUserFromWorkOs, + type WorkOsMembershipPayload, + type WorkOsUserPayload, +} from "./workos-mirror"; + +/** + * A membership as WorkOS lists it for a user: carries the organization's name, + * which is what lets the sign-in feeder mirror the org row without a + * `getOrganization` call. `OrganizationMembership` from the SDK satisfies it. + */ +export interface WorkOsSignInMembership extends WorkOsMembershipPayload { + readonly organizationName: string; +} + +/** + * Record a sign-in: the user's profile, then every organization WorkOS lists + * them in (the org row first, so the membership's foreign key holds) and the + * membership itself. Replays converge: every write is guarded on WorkOS + * `updatedAt`, so a second login with the same payload changes nothing. + * + * `fetchedAt` is the instant the membership list was requested — taken + * BEFORE the WorkOS read, so nothing that changed after it can be mistaken + * for older. A membership list names each organization but carries no + * organization timestamp, so `fetchedAt` is the stamp its name is written + * under: a list fetched before a rename (a login that stalled) cannot revert + * the rename after it landed. A membership of an organization the mirror + * holds as deleted is refused by the mirror, and the organization is neither + * re-minted nor renamed: a list fetched before a deletion cannot restore the + * organization after its purge. And a membership stamped before the + * organization's last full scan (`organizations.backfilled_at`) is refused + * too: a list fetched before a revocation and written after the scan that + * found the membership gone cannot reinstate it. + */ +export const mirrorSignIn = Effect.fn("workos_mirror.signIn")(function* ( + user: WorkOsUserPayload, + memberships: readonly WorkOsSignInMembership[], + fetchedAt: Date, +) { + const mirror = yield* WorkOsMirror; + const users = yield* UserStoreService; + yield* mirror.upsertUser(mirrorUserFromWorkOs(user)); + for (const membership of memberships) { + yield* users.use("upsertOrganization", (s) => + s.upsertOrganization({ + id: membership.organizationId, + name: membership.organizationName, + updatedAt: fetchedAt, + }), + ); + yield* mirror.upsertMembership(mirrorMembershipFromWorkOs(membership)); + } +}); + +/** + * Record one membership WorkOS just returned to a write (create, role + * change, invitation acceptance). The organization must already be mirrored; + * every caller has just upserted it or resolved it through the mirror. + */ +export const mirrorMembership = (membership: WorkOsMembershipPayload) => + Effect.flatMap(WorkOsMirror.asEffect(), (mirror) => + mirror.upsertMembership(mirrorMembershipFromWorkOs(membership)), + ); diff --git a/apps/cloud/src/auth/org-api-key-auth.node.test.ts b/apps/cloud/src/auth/org-api-key-auth.node.test.ts index 624641644f..14e2006f55 100644 --- a/apps/cloud/src/auth/org-api-key-auth.node.test.ts +++ b/apps/cloud/src/auth/org-api-key-auth.node.test.ts @@ -83,18 +83,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: `org-slug-${org.id}`, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganization: async (id: string) => ({ id, name: `Org ${id}`, slug: `org-slug-${id}`, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganizationBySlug: async (slug: string) => ({ id: "org_by_slug", name: `Org ${slug}`, slug, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), deleteOrganizationCascade: async () => {}, @@ -133,7 +142,10 @@ describe("org-level API keys", () => { const auth = yield* resolveBearerAuth(bearer("valid_user_key")).pipe(Effect.provide(layers)); expect(isPlatformAuth(auth)).toBe(false); - expect(auth).toMatchObject({ accountId: "user_123", organizationId: "org_123" }); + expect(auth).toMatchObject({ + accountId: "user_123", + organizationId: "org_123", + }); }), ); diff --git a/apps/cloud/src/auth/org-selector-auth.node.test.ts b/apps/cloud/src/auth/org-selector-auth.node.test.ts index aa9f11c024..050b483a0e 100644 --- a/apps/cloud/src/auth/org-selector-auth.node.test.ts +++ b/apps/cloud/src/auth/org-selector-auth.node.test.ts @@ -86,12 +86,18 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: org.id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganization: async (id: string) => ({ id, name: `Org ${id}`, slug: id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), // The URL slug maps to URL_ORG (the member's other org); any other slug @@ -100,6 +106,9 @@ const stubUsers = Layer.succeed(UserStoreService)({ id: slug === URL_SLUG ? URL_ORG : "org_outsider", name: `Org ${slug}`, slug, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), deleteOrganizationCascade: async () => {}, diff --git a/apps/cloud/src/auth/organization.ts b/apps/cloud/src/auth/organization.ts index 5aceb2ab1a..68d70fad28 100644 --- a/apps/cloud/src/auth/organization.ts +++ b/apps/cloud/src/auth/organization.ts @@ -44,7 +44,11 @@ export const resolveOrganization = (organizationId: string) => const workos = yield* WorkOSClient; const fresh = yield* workos.getOrganization(organizationId); return yield* users.use("upsertOrganization", (s) => - s.upsertOrganization({ id: fresh.id, name: fresh.name }), + s.upsertOrganization({ + id: fresh.id, + name: fresh.name, + updatedAt: new Date(fresh.updatedAt), + }), ); }); diff --git a/apps/cloud/src/auth/user-store.ts b/apps/cloud/src/auth/user-store.ts index 997d6ebc86..931d4fcc80 100644 --- a/apps/cloud/src/auth/user-store.ts +++ b/apps/cloud/src/auth/user-store.ts @@ -7,7 +7,7 @@ // so domain tables can foreign-key against them and so we can resolve org // metadata without an API call on every request. -import { eq } from "drizzle-orm"; +import { and, eq, isNull, lte, or } from "drizzle-orm"; import { generateOrgSlug } from "@executor-js/api"; @@ -18,6 +18,29 @@ import { purgeOrganizationData } from "../db/org-deletion"; export type Account = typeof accounts.$inferSelect; export type Organization = typeof organizations.$inferSelect; +/** + * An organization as a feeder hands it to the mirror. `updatedAt` is when + * `name` is known to have been the organization's name in WorkOS: the WorkOS + * `updatedAt` of an organization payload, or the instant a membership list + * naming the organization was fetched (a list carries the name but no + * organization timestamp). + */ +export interface OrganizationPayload { + readonly id: string; + readonly name: string; + readonly updatedAt: Date; +} + +/** + * Which stored organization rows a name stamped `updatedAt` may rename: a + * row with no stamp (predating the stamp), or one stamped at or before + * `updatedAt` — feeders replay the same payload and must converge. Every + * writer of `organizations.name` applies this, so a name fetched before a + * rename can never revert the rename after it landed. + */ +export const organizationAcceptsName = (updatedAt: Date) => + or(isNull(organizations.workosUpdatedAt), lte(organizations.workosUpdatedAt, updatedAt)); + export const makeUserStore = (db: DrizzleDb) => { const getOrganization = async (id: string) => { const rows = await db.select().from(organizations).where(eq(organizations.id, id)); @@ -38,10 +61,15 @@ export const makeUserStore = (db: DrizzleDb) => { // candidate was claimed by a different org). Returns the inserted row, or // null when either conflict swallowed the insert — the caller decides whether // to re-read (id race) or retry with a new candidate (slug race). - const tryInsertOrg = async (id: string, name: string, slug: string) => { + const tryInsertOrg = async (org: OrganizationPayload, slug: string) => { const [row] = await db .insert(organizations) - .values({ id, name, slug }) + .values({ + id: org.id, + name: org.name, + slug, + workosUpdatedAt: org.updatedAt, + }) .onConflictDoNothing() .returning(); return row ?? null; @@ -49,20 +77,27 @@ export const makeUserStore = (db: DrizzleDb) => { // Every new org row is born with a slug — there is no nullable window and no // self-healing. Existing rows keep their slug (stable across renames, so org - // URLs survive) and only refresh their name. - const upsertOrganization = async (org: { id: string; name: string }) => { + // URLs survive) and only refresh their name — and only from a payload at + // least as new as the one that last named it (`organizationAcceptsName`): + // a sign-in whose membership list was fetched before a rename would + // otherwise revert the rename after it landed. A row marked deleted is + // returned as it is: the organization is gone, and nothing a feeder still + // holds about it (a name, a membership fetched before the deletion) is + // written — never re-minted live, never renamed. + const upsertOrganization = async (org: OrganizationPayload) => { const existing = await getOrganization(org.id); if (existing) { + if (existing.deletedAt !== null) return existing; const [updated] = await db .update(organizations) - .set({ name: org.name }) - .where(eq(organizations.id, org.id)) + .set({ name: org.name, workosUpdatedAt: org.updatedAt }) + .where(and(eq(organizations.id, org.id), organizationAcceptsName(org.updatedAt))) .returning(); return updated ?? existing; } for (let attempt = 0; attempt < 4; attempt++) { const slug = await generateOrgSlug(org.name, slugTaken); - const inserted = await tryInsertOrg(org.id, org.name, slug); + const inserted = await tryInsertOrg(org, slug); if (inserted) return inserted; // The insert was swallowed by a conflict. If the id now exists, a // concurrent request mirrored it — return that row. Otherwise the slug @@ -98,9 +133,11 @@ export const makeUserStore = (db: DrizzleDb) => { return rows[0] ?? null; }, - // Permanently delete an org and everything it owns (tenant data, secrets, - // identity mirror + cascaded memberships) in a single transaction. Callers - // sequence the external WorkOS/Autumn deletions around this. - deleteOrganizationCascade: (id: string) => purgeOrganizationData(db, id), + // Permanently delete everything an org owns (tenant data, secrets, its + // memberships) in a single transaction, leaving the organization row as + // a tombstone marked `deletedAt` (see `purgeOrganizationData` for why). + // Callers sequence the external WorkOS/Autumn deletions around this. + deleteOrganizationCascade: (id: string, deletedAt: Date) => + purgeOrganizationData(db, id, deletedAt), }; }; diff --git a/apps/cloud/src/auth/workos-callback-state.node.test.ts b/apps/cloud/src/auth/workos-callback-state.node.test.ts index 83fd945257..44152dda4e 100644 --- a/apps/cloud/src/auth/workos-callback-state.node.test.ts +++ b/apps/cloud/src/auth/workos-callback-state.node.test.ts @@ -20,6 +20,7 @@ import { CloudAuthPublicHandlers } from "./handlers"; import { CloudAuthPublicApi } from "./api"; import { UserStoreService } from "./context"; import { WorkOSClient, type WorkOSClientService } from "./workos"; +import { WorkOsMirror } from "./workos-mirror"; import { encodeLoginState } from "./login-state"; import { AutumnService } from "../extensions/billing/service"; @@ -79,18 +80,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: org.id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt: new Date(), }), getOrganization: async (id: string) => ({ id, name: "Org " + id, slug: id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt: new Date(), }), getOrganizationBySlug: async (slug: string) => ({ id: slug, name: slug, slug, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt: new Date(), }), deleteOrganizationCascade: async () => {}, @@ -98,6 +108,23 @@ const stubUsers = Layer.succeed(UserStoreService)({ ), }); +// The callback records the sign-in (user + memberships) in the membership +// mirror; every other mirror operation is out of this route's reach. +const stubMirror = Layer.succeed(WorkOsMirror)({ + upsertUser: () => Effect.succeed(true), + upsertMembership: () => Effect.succeed(true), + deleteMembership: () => Effect.die("the callback does not delete memberships"), + deleteUser: () => Effect.die("the callback does not delete users"), + getCursor: () => Effect.die("the callback does not read the events cursor"), + setCursor: () => Effect.die("the callback does not move the events cursor"), + applyOrganizationScan: () => Effect.die("the callback does not run the backfill"), + replayBoundary: () => Effect.die("the callback does not run the reconciler"), + setReplayBoundary: () => Effect.die("the callback does not run the backfill"), + backfillCompletedAt: () => Effect.die("the callback does not check mirror readiness"), + markBackfillCompleted: () => Effect.die("the callback does not run the backfill"), + organizationBackfilledAt: () => Effect.die("the callback does not report seats"), +}); + // Only the public group is under test; the session group (and its SessionAuth // middleware, which needs a live DB) is out of scope — the callback route lives // in CloudAuthPublicApi and requires no middleware. @@ -107,6 +134,7 @@ const App = HttpApiBuilder.layer(PublicApi).pipe( Layer.provide(CloudAuthPublicHandlers), Layer.provide(stubWorkOS), Layer.provide(stubUsers), + Layer.provide(stubMirror), Layer.provide(AutumnService.Default), Layer.provide(HttpServer.layerServices), ); diff --git a/apps/cloud/src/auth/workos-mirror-backfill.ts b/apps/cloud/src/auth/workos-mirror-backfill.ts new file mode 100644 index 0000000000..adcd7c11f7 --- /dev/null +++ b/apps/cloud/src/auth/workos-mirror-backfill.ts @@ -0,0 +1,274 @@ +// --------------------------------------------------------------------------- +// Backfill of the membership mirror from WorkOS, one organization at a time: +// list EVERY membership WorkOS holds for it — active, pending, and inactive +// alike — fetch each member's user, write both through the mirror's guarded +// upserts, tombstone whatever the mirror still holds that WorkOS no longer +// lists, and mark the organization BACKFILLED as of the listing. Inactive +// memberships are listed on purpose: the scan tombstones every mirrored +// membership its listing lacks, and a tombstone is keyed to the membership +// id for good (`membershipAcceptsPayload`), so a listing that skipped the +// inactive ones would tombstone a membership WorkOS merely deactivated and +// refuse its reactivation — under the same id — forever. Listed with its +// real status it stays an ordinary `inactive` row that a newer payload +// reactivates. The core is a pure function over a `source` +// (WorkOS reads) and the mirror store, so the one-off script +// (`scripts/backfill-workos-mirror.ts`) can wire real clients, the request +// path can wire `WorkOSClient` (an organization whose mark is missing is +// scanned on demand before its seats are counted), and the test can wire +// fakes against the test database. +// +// Idempotent and repairing: the upserts are guarded on WorkOS `updatedAt`, +// so a re-scan over unchanged data writes nothing new, and every membership +// the mirror holds for the org that WorkOS no longer lists is TOMBSTONED +// (`inactive`, `deleted_at` = the time the listing was taken) — so a re-scan +// repairs a stale row instead of leaving it granting access. `dryRun` reads +// everything and writes nothing, so the printed counts are the plan. +// +// One scan is applied in ONE transaction (`WorkOsMirror.applyOrganizationScan`) +// that first moves the organization's `backfilled_at` forward to the +// listing's instant and writes nothing if a LATER listing already did. Two +// scans of the same organization can overlap (the one-off script and an +// on-demand scan from a request, or a stalled script run and its retry), and +// the `updatedAt` guard alone cannot order them: a scan that listed a +// membership, stalled, and resumed after a later scan had found it gone +// would insert it live — the later scan tombstoned nothing, because the row +// was not there to tombstone. Refusing the older listing whole is what keeps +// a membership revoked between the two listings revoked. +// +// Completeness is tracked PER ORGANIZATION (`organizations.backfilled_at`), +// never database-wide: the mark is written only by a scan that listed that +// organization's memberships in full, so an organization mirrored after a +// backfill ran (lazily by a request, or by a sign-in that records only the +// caller's own membership) starts unmarked and is scanned before any count +// read from the mirror is trusted. A run over every organization +// (`backfillWorkOsMirror`) additionally records the Events API replay +// boundary — the instant it began reading WorkOS, taken BEFORE anything is +// listed so no change can fall between the boundary and a listing — once +// every organization was written, and only if no boundary is recorded yet. +// A run that fails part-way records nothing, and a later completed run +// keeps the first boundary: a scan refreshes memberships and tombstones, +// not organization names or deleted users' profiles, so an +// `organization.updated` or `user.deleted` between two runs is covered only +// by the events stream — advancing the boundary past it would skip it for +// good. The reconciler's own cursor takes over from the boundary after its +// first page, so the boundary's only job is to name where that first page +// starts. +// +// The mark also orders every OTHER membership write against the scan: the +// mirror refuses a membership payload stamped before the organization's +// `backfilled_at` (`upsertMembership`). A login whose membership list was +// fetched before a revocation and written after the scan would otherwise +// reinstate the revoked membership — and that revocation predates the +// events replay boundary, so no event would ever tombstone it again. +// --------------------------------------------------------------------------- + +import { Clock, Effect, Option } from "effect"; + +import { + mirrorMembershipFromWorkOs, + mirrorUserFromWorkOs, + type WorkOsMembershipPayload, + type WorkOsMirrorShape, + type WorkOsOrganizationScanWrites, + type WorkOsUserPayload, +} from "./workos-mirror-store"; + +/** The WorkOS reads one organization's scan performs, over whatever client the caller wires. */ +export interface WorkOsOrganizationScanSource { + /** + * EVERY membership of one organization, all pages and all statuses + * (active, pending, inactive). A source that filtered by status would + * have the scan tombstone what it filtered out — see the header. + */ + readonly listOrgMembers: ( + organizationId: string, + ) => Effect.Effect; + readonly getUser: (userId: string) => Effect.Effect; +} + +/** The reads the full backfill performs: every organization, then each one's scan. */ +export interface WorkOsMirrorBackfillSource extends WorkOsOrganizationScanSource { + /** Every organization id the mirror knows (FK target of `memberships`). */ + readonly listOrganizationIds: () => Effect.Effect; +} + +export interface WorkOsMirrorBackfillOptions { + readonly dryRun: boolean; + /** One line per organization and one summary line; never a user's data. */ + readonly log: (line: string) => void; +} + +/** What one organization's scan did. */ +export interface WorkOsOrganizationScanCounts extends WorkOsOrganizationScanWrites { + /** Memberships WorkOS reported for the organization. */ + readonly memberships: number; + /** + * Whether the listing was written to the mirror. `false` on a dry run, and + * when the mirror refused the listing whole: a later listing of the + * organization had already been applied (an overlapping scan finished + * first), or the organization is marked deleted or not mirrored. Every + * write count is 0 then. + */ + readonly applied: boolean; +} + +const NOTHING_WRITTEN: WorkOsOrganizationScanWrites = { + usersWritten: 0, + membershipsWritten: 0, + membershipsTombstoned: 0, +}; + +export interface WorkOsMirrorBackfillCounts extends WorkOsOrganizationScanWrites { + readonly organizations: number; + /** Memberships WorkOS reported across every organization. */ + readonly memberships: number; +} + +// Bounded fan-out for the per-member `getUser` calls: enough to overlap the +// WorkOS round-trips, low enough to stay clear of its rate limit. +const USER_FETCH_CONCURRENCY = 5; + +const now = () => Effect.map(Clock.currentTimeMillis, (millis) => new Date(millis)); + +/** + * Scan one organization: list every membership WorkOS holds for it, fetch + * each member's user, and apply the listing to the mirror in one transaction + * — the listed users and memberships upserted, the rest tombstoned, the + * organization marked backfilled as of the listing — unless a later listing + * was applied first, in which case nothing is written (`applied: false`). + * The organization row must already be mirrored. Fails on the first source + * or mirror failure and writes nothing then — a failed scan is safe to + * repeat, so surfacing the failure beats a silent skip. A dry run reads + * everything, writes nothing, and marks nothing. + */ +export const backfillOrganization = ( + source: WorkOsOrganizationScanSource, + mirror: WorkOsMirrorShape, + organizationId: string, + options: { readonly dryRun: boolean }, +) => + Effect.gen(function* () { + // The listing's own instant, taken BEFORE the read so no change can fall + // between them: the tombstone time for whatever the listing no longer + // contains, the cut-off for what may be tombstoned at all (a row stamped + // at or after it was written after the listing and is not missing from + // it), and the organization's new `backfilled_at`. + const listedAt = yield* now(); + const listed = yield* source.listOrgMembers(organizationId); + const members = yield* Effect.forEach( + listed, + (membership) => + Effect.map(source.getUser(membership.userId), (user) => ({ + user: mirrorUserFromWorkOs(user), + membership: mirrorMembershipFromWorkOs(membership), + })), + { concurrency: USER_FETCH_CONCURRENCY }, + ); + if (options.dryRun) { + const counts: WorkOsOrganizationScanCounts = { + ...NOTHING_WRITTEN, + memberships: members.length, + applied: false, + }; + return counts; + } + const written = yield* mirror.applyOrganizationScan({ + organizationId, + listedAt, + members, + }); + if (Option.isNone(written)) { + yield* Effect.logWarning( + "workos_mirror: organization scan not applied — a later listing was already applied, or the organization is deleted or not mirrored", + { organizationId, listedAt: listedAt.toISOString() }, + ); + } + const counts: WorkOsOrganizationScanCounts = { + ...Option.getOrElse(written, () => NOTHING_WRITTEN), + memberships: members.length, + applied: Option.isSome(written), + }; + return counts; + }).pipe( + Effect.withSpan("workos_mirror.backfillOrganization", { + attributes: { organizationId }, + }), + ); + +/** + * Run the full backfill: scan every organization the mirror knows, then + * record the Events API replay boundary if none is recorded yet. Fails on + * the first source or mirror failure — the organizations scanned so far stay + * marked (each was covered in full), the boundary is not recorded, and the + * run is safe to repeat. + */ +export const backfillWorkOsMirror = ( + source: WorkOsMirrorBackfillSource, + mirror: WorkOsMirrorShape, + options: WorkOsMirrorBackfillOptions, +) => + Effect.gen(function* () { + // Taken before the first listing; recorded only once the run completes. + const boundary = yield* now(); + + const organizationIds = yield* source.listOrganizationIds(); + let memberships = 0; + let usersWritten = 0; + let membershipsWritten = 0; + let membershipsTombstoned = 0; + + for (const organizationId of organizationIds) { + const scanned = yield* backfillOrganization(source, mirror, organizationId, options); + memberships += scanned.memberships; + usersWritten += scanned.usersWritten; + membershipsWritten += scanned.membershipsWritten; + membershipsTombstoned += scanned.membershipsTombstoned; + options.log( + `${organizationId} ${scanned.memberships} membership(s)` + + (options.dryRun + ? "" + : scanned.applied + ? ` wrote ${scanned.usersWritten} user(s), ${scanned.membershipsWritten} membership(s), tombstoned ${scanned.membershipsTombstoned}` + : " not applied (a later listing was already applied, or the organization is deleted)"), + ); + } + + const counts: WorkOsMirrorBackfillCounts = { + organizations: organizationIds.length, + memberships, + usersWritten, + membershipsWritten, + membershipsTombstoned, + }; + options.log( + options.dryRun + ? `dry run — ${counts.organizations} organization(s), ${counts.memberships} membership(s) would be mirrored` + : `${counts.organizations} organization(s), ${counts.memberships} membership(s): wrote ${counts.usersWritten} user(s), ${counts.membershipsWritten} membership(s), tombstoned ${counts.membershipsTombstoned}`, + ); + if (!options.dryRun) { + // Every organization was read and written without failure (a failure + // above fails the whole run) — or refused in favour of a listing taken + // later still — so everything before `boundary` is now covered: record + // it for the reconciler — unless an earlier run already did, in which + // case the events between the two runs are the reconciler's to replay + // and the earlier boundary stands. + const recorded = yield* mirror.setReplayBoundary(boundary); + options.log( + recorded + ? `events replay boundary set to ${boundary.toISOString()}` + : "events replay boundary already recorded by an earlier run; kept (the events reconciler replays every change since it)", + ); + // And every live organization is now covered: the mirror is complete + // enough to authorize from (once the reconciler has caught up too), + // which the authorization path reads as the first half of readiness. + // Once: a re-run keeps the first completion. + const completedAt = yield* now(); + const marked = yield* mirror.markBackfillCompleted(completedAt); + options.log( + marked + ? `backfill completion recorded at ${completedAt.toISOString()}` + : "backfill completion already recorded by an earlier run; kept", + ); + } + return counts; + }); diff --git a/apps/cloud/src/auth/workos-mirror-store.ts b/apps/cloud/src/auth/workos-mirror-store.ts new file mode 100644 index 0000000000..e9c7e544b3 --- /dev/null +++ b/apps/cloud/src/auth/workos-mirror-store.ts @@ -0,0 +1,999 @@ +// --------------------------------------------------------------------------- +// The membership mirror's WRITE store — the Drizzle queries behind +// `WorkOsMirror`, plus the converters from WorkOS SDK payloads to mirror rows. +// +// Kept free of `cloudflare:workers` (no `DbService`, no `env`) so the one-off +// backfill (`scripts/backfill-workos-mirror.ts`) can run the SAME upserts over +// a plain postgres.js connection under bun. The request-scoped service that +// wraps this store is `workos-mirror.ts`. +// +// Every write is idempotent and out-of-order safe. Both upserts carry the +// WorkOS `updatedAt` of their payload and refuse to overwrite a row whose +// stored `workos_updated_at` is newer, so a replayed or late-arriving event +// can never regress the mirror. A delete never drops the row: it TOMBSTONES +// it (`status = 'inactive'`, `deleted_at` set) — and INSERTS the tombstone +// when the row is not there yet — so a feeder that fetched the membership +// before the deletion and writes it after (a login, the backfill) finds the +// tombstone instead of reinstating access. The tombstone is protected by +// IDENTITY, not by time: WorkOS never reuses a deleted `om_…` id, so a +// payload naming the tombstoned id is refused whatever it is stamped — a +// role change issued before the removal and delivered after it is stamped +// NEWER than the row and would beat any timestamp guard — while a payload +// naming a DIFFERENT id for the same (account, organization) is the member +// re-added in WorkOS, a replacement, and takes the row over under the usual +// `updatedAt` rule (WorkOS creates it after the old one is deleted, so it is +// always the newer). A tombstone keeps the stamp the row holds, the deleted +// membership's last reported state — never a local clock: read after WorkOS +// answered, it could post-date a replacement created meanwhile and refuse +// it for good. `deleted_at` is the deletion instant when the caller holds +// one (an event's `createdAt`, a scan's listing time) and `now()` otherwise; +// it records WHEN, it orders nothing. A membership WorkOS merely +// deactivated (`status = 'inactive'`, no `deleted_at`) is not a tombstone: +// WorkOS can reactivate it under the same id, and the `updatedAt` rule +// orders that as any other update. The row tombstone can only speak for the +// id the row happens to hold, so every delete ALSO records the deleted id in +// `membership_tombstones`, a ledger keyed by the WorkOS id alone, and every +// membership write consults it first: that covers the case the row cannot — +// membership A replaced by B in WorkOS before the mirror saw either, B then +// deleted. The delete finds a row holding A (not B's to tombstone) and would +// otherwise leave nothing behind; a later scan that still lists B would then +// insert it live, stamped after A and under another id, exactly what the row +// guard lets through. With the ledger the delete is recorded whatever the +// row holds, and B's payload is refused by identity. A scan records the ids +// it tombstones the same way. A deleted USER is protected by identity +// too: `deleteUser` leaves the account row behind as a tombstone (profile +// cleared, stamped with the deletion), and no membership naming that account +// is written again, however the payload is stamped — WorkOS never reuses a +// user id, and a membership the mirror had not seen has no row of its own +// for a guard to refuse the insert against. The cursor advances only by +// compare-and-set, so two reconciler runs cannot both own the stream. +// +// A backfill SCAN of one organization (its full membership listing, taken +// at one instant) is applied in ONE transaction that first compare-and-sets +// the organization's `backfilled_at` to the listing's instant +// (`applyOrganizationScan`): the row stays locked until commit, so two +// overlapping scans serialize on it, and the one whose listing is older +// than the recorded one writes NOTHING. The `updatedAt` guard alone cannot +// order two scans: a scan that listed a membership, stalled, and resumed +// after a later scan had found it gone would insert it live — the later scan +// left no tombstone, because the row was not there to tombstone. For the +// same reason every OTHER membership write is ordered against the scan too: +// a payload stamped before the organization's `backfilled_at` is refused +// unless the row already holds it — judged inside the write's own +// transaction, reading the organization row `FOR SHARE`, so a write that +// races a scan waits for the scan's commit and sees the mark it set, and +// never inserts a membership the scan has just proved revoked. A completed +// scan is the full listing as +// of that instant, so a membership it did not contain but an older payload +// still names (a login whose list was fetched before the revocation and +// written after the scan) was revoked before the scan — and that revocation +// predates the events replay boundary, so nothing would ever tombstone the +// reinstated row. The scan's own writes are the one exception: they are the +// listing that sets the mark. +// +// The events replay boundary (`workos_sync.range_start`) is written ONCE, by +// the first completed backfill, and never advanced: a later backfill +// refreshes memberships only, so an organization rename or user deletion +// between two runs is covered by the events stream alone, and moving the +// boundary past it would skip it for good. +// --------------------------------------------------------------------------- + +import { and, eq, isNotNull, isNull, lt, ne, notInArray, or, sql } from "drizzle-orm"; +import type { AnyPgColumn } from "drizzle-orm/pg-core"; +import { Effect, Option } from "effect"; + +import type { MemberStatus } from "@executor-js/api/server"; + +import { + accounts, + membershipTombstones, + memberships, + organizations, + workosSync, +} from "../db/schema"; +import type { DrizzleDb } from "../db/db"; +import { + WorkOsMirrorError, + tryPromiseService, + userStoreReasonFromCause, + withServiceLogging, +} from "./errors"; + +/** A WorkOS user, as the mirror stores it. `updatedAt` is WorkOS's own. */ +export interface WorkOsMirrorUser { + readonly id: string; + readonly email: string; + readonly firstName: string | null; + readonly lastName: string | null; + readonly avatarUrl: string | null; + readonly lastSignInAt: Date | null; + readonly updatedAt: Date; +} + +/** + * A WorkOS organization membership, as the mirror stores it. `id` is the + * WorkOS `om_…`; `accountId` the WorkOS user id; `updatedAt` is WorkOS's own. + * The organization row must already be mirrored (`upsertOrganization`) — a + * membership of an unknown org is a `query` failure, not a silent skip. + */ +export interface WorkOsMirrorMembership { + readonly id: string; + readonly accountId: string; + readonly organizationId: string; + readonly role: string; + readonly status: MemberStatus; + readonly updatedAt: Date; +} + +/** + * What identifies a membership to a delete: the WorkOS `om_…` id AND the + * (account, organization) pair it belongs to. The pair is the row's key, + * and a delete must be able to mint the row as a tombstone when the mirror + * has not seen the membership yet — an id alone cannot. + */ +export type WorkOsMirrorMembershipRef = Pick< + WorkOsMirrorMembership, + "id" | "accountId" | "organizationId" +>; + +/** One member as a backfill scan lists it: the membership and its user. */ +export interface WorkOsScannedMember { + readonly user: WorkOsMirrorUser; + readonly membership: WorkOsMirrorMembership; +} + +/** + * One organization's full membership listing, as `applyOrganizationScan` + * applies it. `listedAt` is the instant the listing was taken — BEFORE the + * WorkOS read, so no change can fall between the instant and the listing — + * and is the tombstone time for what the listing no longer contains, the + * cut-off for what may be tombstoned at all, and the organization's new + * `backfilled_at`. + */ +export interface WorkOsOrganizationScan { + readonly organizationId: string; + readonly listedAt: Date; + readonly members: readonly WorkOsScannedMember[]; +} + +/** What an applied scan wrote: the upserts the `updatedAt` guard let through, and the tombstones. */ +export interface WorkOsOrganizationScanWrites { + readonly usersWritten: number; + readonly membershipsWritten: number; + readonly membershipsTombstoned: number; +} + +export interface WorkOsMirrorShape { + /** + * Insert or refresh a user row. `false` when the payload was refused and + * the row left untouched: the stored row is newer than `updatedAt`, or is + * a deletion tombstone stamped at `updatedAt` or later. + */ + readonly upsertUser: (user: WorkOsMirrorUser) => Effect.Effect; + /** + * Insert or refresh a membership row, minting the bare account row first so + * the foreign key holds when the membership arrives before its user. `false` + * when the payload was refused: the stored row is newer than `updatedAt`, + * or is a deletion tombstone of THIS membership id, or THIS membership id + * is in the deletion ledger (`membership_tombstones`, written by every + * delete and every scan tombstone, whatever row it found) — a deleted + * WorkOS id never returns, however the payload is stamped; a payload under + * another id is a replacement and is ordered by `updatedAt` against the + * row's stamp — or the account is a deletion tombstone (`deleteUser`), whatever + * the payload is stamped: WorkOS never reuses a user id, so a deleted user + * has no memberships to mirror — or the organization is marked deleted + * (`organizations.deleted_at`), in which case nothing is written at all: + * no membership of a deleted organization is ever mirrored — or the + * payload is stamped BEFORE the organization's last full scan + * (`organizations.backfilled_at`): the scan listed everything WorkOS held + * at that instant, so a membership it wrote already carries a stamp at + * least this new (the write would change nothing) and one it did not + * write was gone by then and must not come back from a list fetched + * earlier. Only the scan itself writes past that mark + * (`applyOrganizationScan`). The organization checks and the write run in + * ONE transaction that holds the organization row `FOR SHARE`, so a scan + * claiming the row at the same time is waited for and its mark seen — and + * the account row `FOR SHARE` too, so a `deleteUser` tombstoning it at the + * same time is waited for and its tombstone seen. + */ + readonly upsertMembership: ( + membership: WorkOsMirrorMembership, + ) => Effect.Effect; + /** + * Tombstone the membership as deleted: the row stays (or is minted, when + * the mirror has not seen the membership yet), `inactive`, carrying the + * deleted WorkOS id with `deleted_at` set. The tombstone is protected by + * that id: `upsertMembership` refuses every later payload naming it, + * however stamped, and accepts only a REPLACEMENT under another id. The + * row's `workos_updated_at` is left as it is — the deleted membership's + * last reported state, which a replacement is always stamped after; a + * local clock read after WorkOS answered could post-date a replacement + * created meanwhile and must never become the row's stamp. `deletedAt` + * is the deletion instant when the caller holds one (a deletion event's + * `createdAt`, a scan's listing time) or `null` when it holds none + * (WorkOS answers a delete with no time): `deleted_at` then takes the + * current time. It records when the membership was deleted; it orders + * nothing. Matches the row by IDENTITY: `false` when the row is already + * tombstoned (a replayed delete), or holds ANOTHER membership id — the + * member re-added in WorkOS under a new id, which stands whether the + * replacement was mirrored before or after this delete. In EVERY case the + * deleted id is recorded in the deletion ledger (`membership_tombstones`), + * so a membership the mirror never held under its own id (replaced and + * deleted in WorkOS before the mirror saw it) cannot be inserted live by a + * scan that listed it before the deletion: `true` when the row was + * tombstoned OR the id was newly recorded. The organization row must + * already be mirrored, as for `upsertMembership`. + */ + readonly deleteMembership: ( + membership: WorkOsMirrorMembershipRef, + deletedAt: Date | null, + ) => Effect.Effect; + /** + * Tombstone a deleted WorkOS user: every membership of the account is + * tombstoned as by `deleteMembership` (as of `deletedAt`; a deleted user's + * membership ids never return), and the account row is kept (it anchors + * foreign keys) — or minted, when the mirror has not seen the user yet — + * with its profile cleared and stamped `deletedAt`, so a stale user + * payload cannot restore it. `false` when the account already carries + * this tombstone or a later one (a replayed delete). ONE transaction that + * locks the account row first (`FOR NO KEY UPDATE`) and holds it until + * the memberships are tombstoned: `upsertMembership` reads that row `FOR + * SHARE` before it inserts, so a membership write racing the deletion + * waits for it and sees the tombstone, or committed first and is + * tombstoned here — never a live membership left behind for a deleted + * user. + */ + readonly deleteUser: ( + accountId: string, + deletedAt: Date, + ) => Effect.Effect; + /** The id of the last WorkOS event applied, or `null` before the first run. */ + readonly getCursor: () => Effect.Effect; + /** + * Compare-and-set the cursor: advance to `next` only if it still reads + * `prev` (`null` = no cursor yet). `false` means another run moved it first + * — the caller must stop, it no longer owns the stream. + */ + readonly setCursor: ( + prev: string | null, + next: string, + ) => Effect.Effect; + /** + * Apply one organization's backfill scan — the memberships (with their + * users) a WorkOS listing taken at `listedAt` contained — atomically: in a + * single transaction, compare-and-set the organization's `backfilled_at` + * forward to `listedAt`, and only if that succeeded upsert every listed + * user and membership and tombstone (as `deleteMembership` does, at + * `listedAt`) every membership of the organization the listing did NOT + * contain: the rows whose WorkOS id is not among the listed ones AND whose + * stamp is older than the listing. A row stamped at or after `listedAt` + * was written after the listing was taken (it could not be in it) and is + * left alone — its own event lands through the reconciler; a tombstone + * here would beat that event. Rows already tombstoned are left alone. + * + * The organization row stays locked until commit, so two overlapping + * scans serialize on it and the one whose listing is older than (or the + * same instant as) the recorded one writes nothing: `None` — the mirror + * already holds the organization as of a later listing, and a membership + * that listing no longer contained must not be inserted from this one. + * `None` too for an organization the mirror does not hold or has marked + * deleted: there is nothing to scan into. `Some` carries what was written. + */ + readonly applyOrganizationScan: ( + scan: WorkOsOrganizationScan, + ) => Effect.Effect, WorkOsMirrorError>; + /** + * The Events API replay boundary: the instant the FIRST completed one-off + * backfill began reading WorkOS, or `null` if none has completed. The + * reconciler's first run (no cursor yet) reads the stream from here — the + * backfill covers everything before it — and without a boundary it must + * not guess. + */ + readonly replayBoundary: () => Effect.Effect; + /** + * Record the replay boundary, ONCE: `at` is the instant a completed + * backfill began reading WorkOS, and it is kept only when no boundary is + * recorded yet — `true` when this call recorded it. A later completed run + * never moves it: the backfill refreshes memberships and tombstones only, + * not organization names or deleted users' profiles, so a change between + * two runs is covered only by the events stream, which must still be read + * from the first boundary. Written only after every organization has been + * written, so a run that fails part-way records nothing. Never touches the + * cursor: a stream already being followed keeps its position, and the + * boundary is then unused. + */ + readonly setReplayBoundary: (at: Date) => Effect.Effect; + /** + * When a backfill run first wrote EVERY live organization + * (`workos_sync.backfill_completed_at`), or `null` while none has. The + * first half of the mirror's READINESS for authorization: until a run has + * covered every organization, the mirror may lack members who have not + * signed in since it shipped, and a membership check read from it would + * deny them. The other half is the events reconciler being caught up, + * which its cursor row reports. + */ + readonly backfillCompletedAt: () => Effect.Effect; + /** + * Record that a backfill run has written every live organization, as of + * `at` — ONCE: a later completed run keeps the first mark (`false`), so + * readiness never flips back. Written only after the last organization + * was applied (or refused in favour of a later listing), so a run that + * fails part-way records nothing here. Mints the events row when absent, + * as `setReplayBoundary` does, and touches neither the cursor nor the + * boundary. + */ + readonly markBackfillCompleted: (at: Date) => Effect.Effect; + /** + * When the organization's membership list was last FULLY scanned from + * WorkOS (`backfillOrganization` in workos-mirror-backfill.ts), or `null` + * if it never was — or the organization is not mirrored. Until it has been, + * the mirror may hold only the members login and write-through happened to + * record, so a member count read from it is PARTIAL: every seat gate reads + * this first and scans the organization when it is `null`. Per + * organization, never database-wide, so an organization mirrored after a + * backfill ran (lazily, or by a sign-in) is never mistaken for a scanned + * one. + */ + readonly organizationBackfilledAt: ( + organizationId: string, + ) => Effect.Effect; +} + +// --------------------------------------------------------------------------- +// SDK payload → mirror row. The feeders (login callback, write-through, the +// backfill, the Events reconciler) all hand the mirror WorkOS objects; this is +// the one place their field names and ISO timestamps are translated. Typed +// structurally (the fields actually read) so the SDK's `User` / +// `OrganizationMembership`, an event payload, and a test fixture all fit. +// --------------------------------------------------------------------------- + +/** The WorkOS user fields the mirror reads. `User` from the SDK satisfies it. */ +export interface WorkOsUserPayload { + readonly id: string; + readonly email: string; + readonly firstName: string | null; + readonly lastName: string | null; + readonly profilePictureUrl: string | null; + readonly lastSignInAt: string | null; + readonly updatedAt: string; +} + +/** + * The WorkOS membership fields the mirror reads. `OrganizationMembership` from + * the SDK satisfies it; its `status` is exactly the mirror's `MemberStatus`. + */ +export interface WorkOsMembershipPayload { + readonly id: string; + readonly userId: string; + readonly organizationId: string; + readonly role: { readonly slug: string }; + readonly status: MemberStatus; + readonly updatedAt: string; +} + +/** Translate a WorkOS user payload to the row `upsertUser` stores. */ +export const mirrorUserFromWorkOs = (user: WorkOsUserPayload): WorkOsMirrorUser => ({ + id: user.id, + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + avatarUrl: user.profilePictureUrl, + lastSignInAt: user.lastSignInAt === null ? null : new Date(user.lastSignInAt), + updatedAt: new Date(user.updatedAt), +}); + +/** Translate a WorkOS membership payload to the row `upsertMembership` stores. */ +export const mirrorMembershipFromWorkOs = ( + membership: WorkOsMembershipPayload, +): WorkOsMirrorMembership => ({ + id: membership.id, + accountId: membership.userId, + organizationId: membership.organizationId, + role: membership.role.slug, + status: membership.status, + updatedAt: new Date(membership.updatedAt), +}); + +/** + * The `workos_sync` row of the one WorkOS events stream the reconciler + * follows. A row id rather than a singleton table so a second stream + * (another WorkOS environment, a replay) can be added without a schema + * change; the same row carries the backfill's replay boundary and + * completion mark. Not a `db/schema.ts` export: that module's exports are + * enumerated as tables by the purge-coverage test. + */ +export const WORKOS_EVENTS_STREAM_ID = "events"; + +// A raw `sql` fragment binds a Date without the column's driver mapping, so +// every instant below is passed as ISO text and cast. +const instant = (at: Date) => sql`${at.toISOString()}::timestamptz`; + +// An account tombstone moves the row's timestamp to the deletion time — +// never backwards, so a row that somehow carries a newer WorkOS timestamp +// keeps it and the upsert guard stays at least as strict. +const noEarlierThan = (column: AnyPgColumn, at: Date) => sql`greatest(${column}, ${instant(at)})`; + +// A membership tombstone keeps the row, marks it `inactive`, and records the +// deletion in `deleted_at`: the instant the caller holds, or `now()` when it +// holds none (see `deleteMembership`). `workos_updated_at` is left as it is +// — the tombstone is protected by the deleted id, not by its stamp. +const tombstone = (deletedAt: Date | null) => ({ + status: "inactive" as const, + deletedAt: deletedAt === null ? sql`now()` : instant(deletedAt), +}); + +// Which stored membership rows a payload for membership `id` stamped +// `updatedAt` may overwrite. A row WorkOS still holds (`deleted_at` null) is +// ordered by time: a row with no stamp (predating the mirror), any row +// stamped earlier, and a LIVE row stamped the same instant — feeders replay +// the same payload and must converge, not stall. An `inactive` row stamped +// the same instant is not overwritten: a deactivation at T beats an active +// payload at T. A deletion tombstone is ordered by IDENTITY: never +// overwritten by a payload naming the deleted id — a deleted WorkOS +// membership id never returns, however the payload is stamped — and taken +// over by a payload naming another id, a replacement membership WorkOS +// created after the deletion and so stamped after the row's last state, +// under the plain timestamp rule. A tombstone with no id at all (a +// pre-mirror row of a deleted user) is never taken over: the user is gone. +const membershipAcceptsPayload = (id: string, updatedAt: Date) => + or( + and( + isNull(memberships.deletedAt), + or( + isNull(memberships.workosUpdatedAt), + lt(memberships.workosUpdatedAt, updatedAt), + and(eq(memberships.workosUpdatedAt, updatedAt), ne(memberships.status, "inactive")), + ), + ), + and( + isNotNull(memberships.deletedAt), + ne(memberships.membershipId, id), + or(isNull(memberships.workosUpdatedAt), lt(memberships.workosUpdatedAt, updatedAt)), + ), + ); + +// A deleted user's account row, as `deleteUser` leaves it: the profile is +// cleared (every WorkOS user payload carries an email, so a stamped row with +// none was written by `deleteUser`) and the stamp is the deletion time. A row +// minted bare by `ensureAccount` has no stamp either, and is not a tombstone. +// Judged in code, over a row read under a lock (`writeMembership`), not as a +// predicate in the read: a `SELECT ... FOR SHARE` locks only the rows it +// returns, so a read filtered to tombstones would lock nothing for a live +// row — the one case the lock exists for. +const isAccountTombstone = (account: { + readonly email: string | null; + readonly workosUpdatedAt: Date | null; +}): boolean => account.email === null && account.workosUpdatedAt !== null; + +// The same rule as for a membership row, for an account row. A tombstone +// takes no payload stamped at or before the deletion; a bare row takes any. +const accountAcceptsPayload = (updatedAt: Date) => + or( + isNull(accounts.workosUpdatedAt), + lt(accounts.workosUpdatedAt, updatedAt), + and(eq(accounts.workosUpdatedAt, updatedAt), isNotNull(accounts.email)), + ); + +// A delete is applied only to a row not yet tombstoned: a replayed deletion +// changes nothing and reports so. +const notDeleted = isNull(memberships.deletedAt); + +// Whether membership `id` is in the deletion ledger: a WorkOS id a delete or +// a scan has named as gone, which never returns. Identity alone — the +// ledger records WHEN for the record, not for ordering. +const membershipIdDeleted = async (db: DrizzleDb, id: string): Promise => { + const rows = await db + .select({ membershipId: membershipTombstones.membershipId }) + .from(membershipTombstones) + .where(eq(membershipTombstones.membershipId, id)); + return rows.length > 0; +}; + +// Which (account, organization) row a delete of membership `id` may +// tombstone: the row carrying THIS id — whatever its stamp, a deleted id is +// never reused — or a row with no id at all (written before the mirror +// recorded WorkOS ids; the delete fills the id in). Never a row under +// ANOTHER id: that is a different membership of the same account and +// organization, the member re-added in WorkOS after this one was removed, +// and it stands however the two are stamped. Identity orders them, +// timestamps do not. A row already tombstoned is left alone (`notDeleted`) +// so a replayed delete reports `false`. +const membershipDeletableBy = (id: string) => + and(or(isNull(memberships.membershipId), eq(memberships.membershipId, id)), notDeleted); + +// The write queries, over `db` or over a transaction handle (drizzle's is a +// `PgDatabase` too): the one `applyOrganizationScan` opens, or the one the +// store opens per `upsertMembership`. Each answers whether it wrote a row; +// the public shape translates that. +const makeWrites = (db: DrizzleDb) => { + const ensureAccount = (id: string) => + db.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id }); + + // The membership upsert under the row guard (`membershipAcceptsPayload`) + // and the account tombstone, never the organization mark. + const writeMembership = async (membership: WorkOsMirrorMembership): Promise => { + await ensureAccount(membership.accountId); + // Never for a DELETED user. The row guard below orders a payload against + // the membership row it would overwrite; a membership the mirror has not + // seen yet has no row, so the guard cannot refuse the INSERT — and a + // feeder that fetched the membership before the user was deleted and + // writes it after (a stalled login list, the backfill's older listing) + // would insert it live. The account tombstone is the one row a deleted + // user always leaves behind, so it is consulted first, by identity: + // WorkOS never reuses a user id, so no payload naming a tombstoned + // account is ever current. + // + // Read FOR SHARE, held until the transaction `db` belongs to commits: a + // `deleteUser` applying the deletion at this moment holds the row FOR NO + // KEY UPDATE until ITS commit, so this read waits for it and sees the + // tombstone — and a deletion that arrives after this read waits for this + // transaction, then tombstones the membership it inserted. Unlocked, a + // deletion could land between this read and the insert below and leave + // a live membership for a deleted user. + const locked = await db + .select({ email: accounts.email, workosUpdatedAt: accounts.workosUpdatedAt }) + .from(accounts) + .where(eq(accounts.id, membership.accountId)) + .for("share"); + const account = locked[0]; + // `ensureAccount` guarantees the row; accounts are tombstoned, never + // deleted, so a missing one is refused like a tombstone, not written for. + if (account === undefined || isAccountTombstone(account)) return false; + // Never under a DELETED membership id. The row guard below can only + // refuse against the id the row holds; a delete of THIS id that found + // the row under another id (see the header) left only the ledger entry + // behind, and that is what refuses the payload here. + if (await membershipIdDeleted(db, membership.id)) return false; + const written = await db + .insert(memberships) + .values({ + accountId: membership.accountId, + organizationId: membership.organizationId, + membershipId: membership.id, + role: membership.role, + status: membership.status, + workosUpdatedAt: membership.updatedAt, + }) + .onConflictDoUpdate({ + target: [memberships.accountId, memberships.organizationId], + set: { + membershipId: membership.id, + role: membership.role, + status: membership.status, + workosUpdatedAt: membership.updatedAt, + // A replacement taking over a tombstone is live again; a row that + // was not tombstoned had nothing here. + deletedAt: null, + }, + setWhere: membershipAcceptsPayload(membership.id, membership.updatedAt), + }) + .returning({ accountId: memberships.accountId }); + return written.length > 0; + }; + + return { + upsertUser: async (user: WorkOsMirrorUser): Promise => { + const written = await db + .insert(accounts) + .values({ + id: user.id, + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + avatarUrl: user.avatarUrl, + lastSignInAt: user.lastSignInAt, + workosUpdatedAt: user.updatedAt, + }) + .onConflictDoUpdate({ + target: accounts.id, + set: { + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + avatarUrl: user.avatarUrl, + lastSignInAt: user.lastSignInAt, + workosUpdatedAt: user.updatedAt, + }, + setWhere: accountAcceptsPayload(user.updatedAt), + }) + .returning({ id: accounts.id }); + return written.length > 0; + }, + + upsertMembership: async (membership: WorkOsMirrorMembership): Promise => { + // Never into an organization the mirror holds as DELETED: its + // memberships are gone from WorkOS and purged (or being purged) here, + // so a payload that still names it was fetched before the deletion — + // a login that stalled across the purge, a stale event — and writing + // it would grant access to a deleted organization. The tombstone row + // outlives the purge for exactly this check. And never from a payload + // stamped before the organization's last full scan: the scan is the + // complete listing as of `backfilled_at`, so an older payload either + // repeats a row the scan wrote (nothing to change) or names a + // membership the scan found gone — revoked before the mirror's events + // replay begins, so no event would ever tombstone it again. The row + // is read FOR SHARE, so a scan claiming it at this moment + // (`claimOrganizationScan`, an UPDATE that holds the row until its + // transaction commits) and the purge marking it deleted are waited + // for, and the mark read is the committed one: a scan cannot slip + // between this read and the write below and leave a membership it + // proved revoked inserted after it. The lock is held until the + // transaction `db` belongs to ends — `makeWorkOsMirrorStore` opens one + // per call, `applyPage` runs this on the page's own — which is what + // orders the write after the scan. An organization the mirror does + // not hold at all is still a foreign-key failure in the write. + const organization = await db + .select({ + deletedAt: organizations.deletedAt, + backfilledAt: organizations.backfilledAt, + }) + .from(organizations) + .where(eq(organizations.id, membership.organizationId)) + .for("share"); + const row = organization[0]; + if (row?.deletedAt != null) return false; + if (row?.backfilledAt != null && membership.updatedAt < row.backfilledAt) return false; + return writeMembership(membership); + }, + + // A scan's own membership writes: the scan has just claimed the + // organization as of its listing, so its payloads are exactly what the + // mark stands for and are not ordered against it. + writeScannedMembership: writeMembership, + + // An upsert, like the membership write it guards against: a delete the + // mirror sees before the membership itself (the reconciler ahead of the + // backfill) must leave the tombstone behind, or the later, older payload + // would insert the row live. + deleteMembership: async ( + membership: WorkOsMirrorMembershipRef, + deletedAt: Date | null, + ): Promise => { + await ensureAccount(membership.accountId); + // Lock the account row FOR NO KEY UPDATE, as `deleteUser` does: a + // membership write of this user reads the row FOR SHARE before it + // consults the ledger (`writeMembership`), so a write racing this + // delete either waits here and then finds the ledger entry, or + // committed first and is tombstoned by the row upsert below. Without + // the lock a write could pass the ledger check before this entry + // lands and insert the deleted membership live after it. + await db + .select({ id: accounts.id }) + .from(accounts) + .where(eq(accounts.id, membership.accountId)) + .for("no key update"); + // The ledger entry FIRST, whatever the row holds: the one record of + // the deletion that does not depend on the row carrying the deleted + // id. A replayed delete finds it there and records nothing new. + const recorded = await db + .insert(membershipTombstones) + .values({ + membershipId: membership.id, + accountId: membership.accountId, + organizationId: membership.organizationId, + ...(deletedAt === null ? {} : { deletedAt }), + }) + .onConflictDoNothing({ target: membershipTombstones.membershipId }) + .returning({ membershipId: membershipTombstones.membershipId }); + const tombstoned = await db + .insert(memberships) + .values({ + accountId: membership.accountId, + organizationId: membership.organizationId, + membershipId: membership.id, + ...tombstone(deletedAt), + }) + .onConflictDoUpdate({ + target: [memberships.accountId, memberships.organizationId], + set: { membershipId: membership.id, ...tombstone(deletedAt) }, + setWhere: membershipDeletableBy(membership.id), + }) + .returning({ accountId: memberships.accountId }); + return recorded.length > 0 || tombstoned.length > 0; + }, + + // Only inside a transaction: the account row lock taken first is what + // orders this against a membership write of the same user, and it lasts + // exactly as long as the transaction `db` belongs to — the one the store + // opens per call. + deleteUser: async (accountId: string, deletedAt: Date): Promise => { + // Lock the account row FIRST — minted bare when absent, so there is a + // row to lock (the tombstone is minted for the same reason as the + // membership one: a later, older payload must find it) — and hold it + // FOR NO KEY UPDATE until commit. A membership write reads the row FOR + // SHARE before it inserts (`writeMembership`), so a write racing this + // delete either waits here and then sees the tombstone, or committed + // before this lock was granted and is caught by the membership + // tombstoning below. Without the lock a write could read the row live + // and insert its membership after the tombstoning had run. + await ensureAccount(accountId); + await db + .select({ id: accounts.id }) + .from(accounts) + .where(eq(accounts.id, accountId)) + .for("no key update"); + // The account tombstone: profile cleared, stamped with the deletion. + // Applied unless the row already carries this tombstone or a later one + // (a replayed delete). + const cleared = await db + .update(accounts) + .set({ + email: null, + firstName: null, + lastName: null, + avatarUrl: null, + workosUpdatedAt: noEarlierThan(accounts.workosUpdatedAt, deletedAt), + }) + .where( + and( + eq(accounts.id, accountId), + or( + isNotNull(accounts.email), + isNull(accounts.workosUpdatedAt), + lt(accounts.workosUpdatedAt, deletedAt), + ), + ), + ) + .returning({ id: accounts.id }); + await db + .update(memberships) + .set(tombstone(deletedAt)) + .where(and(eq(memberships.accountId, accountId), notDeleted)); + return cleared.length > 0; + }, + + // Tombstone (at `listedAt`) every membership of the organization that a + // listing taken at `listedAt` did not contain. Only inside a scan's + // transaction, after its `backfilled_at` CAS: on its own this could + // tombstone what a LATER scan just wrote. + tombstoneMembershipsExcept: async ( + organizationId: string, + membershipIds: readonly string[], + listedAt: Date, + ): Promise => { + const tombstoned = await db + .update(memberships) + .set(tombstone(listedAt)) + .where( + and( + eq(memberships.organizationId, organizationId), + // A row with no WorkOS id predates the mirror and is not in + // any listing; if WorkOS still holds the membership, the + // scan's upsert has just filled the id in. + or( + isNull(memberships.membershipId), + notInArray(memberships.membershipId, [...membershipIds]), + ), + // Only rows the listing could have contained: stamped before + // it was taken (or never stamped). Anything newer was written + // after the listing and is not missing from it. + or(isNull(memberships.workosUpdatedAt), lt(memberships.workosUpdatedAt, listedAt)), + notDeleted, + ), + ) + .returning({ + accountId: memberships.accountId, + membershipId: memberships.membershipId, + }); + // The ids the listing proved gone go into the ledger too, as any other + // delete's: they never return. A row with no id has none to record. + const gone = tombstoned.flatMap((row) => + row.membershipId === null + ? [] + : [ + { + membershipId: row.membershipId, + accountId: row.accountId, + organizationId, + deletedAt: listedAt, + }, + ], + ); + if (gone.length > 0) { + await db + .insert(membershipTombstones) + .values(gone) + .onConflictDoNothing({ target: membershipTombstones.membershipId }); + } + return tombstoned.length; + }, + }; +}; + +// Claim the organization for a scan listed at `listedAt`: move its +// `backfilled_at` forward to `listedAt` if the recorded mark is older (or +// missing). Run inside a transaction this also LOCKS the organization row +// until commit, so an overlapping scan's claim waits here, then reads the +// moved mark and matches nothing. An organization marked deleted is never +// claimed: its memberships are being (or have been) purged, and a scan +// that listed them before the deletion must not write them back. +const claimOrganizationScan = async (db: DrizzleDb, organizationId: string, listedAt: Date) => { + const claimed = await db + .update(organizations) + .set({ backfilledAt: listedAt }) + .where( + and( + eq(organizations.id, organizationId), + isNull(organizations.deletedAt), + or(isNull(organizations.backfilledAt), lt(organizations.backfilledAt, listedAt)), + ), + ) + .returning({ id: organizations.id }); + return claimed.length > 0; +}; + +/** + * The mirror's write operations over `db`. Failures are `WorkOsMirrorError` + * naming the operation and the classified driver reason; the full cause is + * logged at the boundary. + */ +export const makeWorkOsMirrorStore = (db: DrizzleDb): WorkOsMirrorShape => { + const run = (op: string, fn: () => Promise) => + withServiceLogging( + `workos_mirror.${op}`, + (failure) => + new WorkOsMirrorError({ + operation: op, + reason: userStoreReasonFromCause(failure), + }), + tryPromiseService(fn), + ); + + const writes = makeWrites(db); + + return { + upsertUser: (user) => run("upsertUser", () => writes.upsertUser(user)), + + // One transaction per call: the organization guard inside locks the org + // row until the write has landed (see `makeWrites`). + upsertMembership: (membership) => + run("upsertMembership", () => + db.transaction((tx) => makeWrites(tx).upsertMembership(membership)), + ), + + // One transaction per call: the ledger entry and the row tombstone land + // together or not at all. + deleteMembership: (membership, deletedAt) => + run("deleteMembership", () => + db.transaction((tx) => makeWrites(tx).deleteMembership(membership, deletedAt)), + ), + + // One transaction per call: the account row lock inside is held until + // the memberships are tombstoned (see `makeWrites`). + deleteUser: (accountId, deletedAt) => + run("deleteUser", () => + db.transaction((tx) => makeWrites(tx).deleteUser(accountId, deletedAt)), + ), + + getCursor: () => + run("getCursor", async () => { + const rows = await db + .select({ cursor: workosSync.cursor }) + .from(workosSync) + .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID)); + // No row yet is the same state as a row with no cursor: nothing applied. + return rows[0]?.cursor ?? null; + }), + + setCursor: (prev, next) => + run("setCursor", async () => { + const now = new Date(); + if (prev === null) { + // First advance: mint the row, or claim an existing row that still + // has no cursor. A row that already carries one belongs to another + // run and is left alone. + const written = await db + .insert(workosSync) + .values({ id: WORKOS_EVENTS_STREAM_ID, cursor: next, updatedAt: now }) + .onConflictDoUpdate({ + target: workosSync.id, + set: { cursor: next, updatedAt: now }, + setWhere: isNull(workosSync.cursor), + }) + .returning({ id: workosSync.id }); + return written.length > 0; + } + const written = await db + .update(workosSync) + .set({ cursor: next, updatedAt: now }) + .where(and(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), eq(workosSync.cursor, prev))) + .returning({ id: workosSync.id }); + return written.length > 0; + }), + + applyOrganizationScan: (scan) => + run("applyOrganizationScan", () => + db.transaction(async (tx) => { + // The claim comes FIRST so the lock is held for every write below; + // a scan that lost to a later listing commits an empty transaction. + const claimed = await claimOrganizationScan(tx, scan.organizationId, scan.listedAt); + if (!claimed) return Option.none(); + const txWrites = makeWrites(tx); + let usersWritten = 0; + let membershipsWritten = 0; + for (const member of scan.members) { + if (await txWrites.upsertUser(member.user)) usersWritten += 1; + if (await txWrites.writeScannedMembership(member.membership)) membershipsWritten += 1; + } + const membershipsTombstoned = await txWrites.tombstoneMembershipsExcept( + scan.organizationId, + scan.members.map((member) => member.membership.id), + scan.listedAt, + ); + const written: WorkOsOrganizationScanWrites = { + usersWritten, + membershipsWritten, + membershipsTombstoned, + }; + return Option.some(written); + }), + ), + + replayBoundary: () => + run("replayBoundary", async () => { + const rows = await db + .select({ rangeStart: workosSync.rangeStart }) + .from(workosSync) + .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID)); + return rows[0]?.rangeStart ?? null; + }), + + setReplayBoundary: (at) => + run("setReplayBoundary", async () => { + const recorded = await db + .insert(workosSync) + .values({ + id: WORKOS_EVENTS_STREAM_ID, + cursor: null, + rangeStart: at, + updatedAt: at, + }) + .onConflictDoUpdate({ + target: workosSync.id, + set: { rangeStart: at }, + // A boundary already recorded stands, whatever this run's is. + setWhere: isNull(workosSync.rangeStart), + }) + .returning({ id: workosSync.id }); + return recorded.length > 0; + }), + + backfillCompletedAt: () => + run("backfillCompletedAt", async () => { + const rows = await db + .select({ backfillCompletedAt: workosSync.backfillCompletedAt }) + .from(workosSync) + .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID)); + return rows[0]?.backfillCompletedAt ?? null; + }), + + markBackfillCompleted: (at) => + run("markBackfillCompleted", async () => { + const recorded = await db + .insert(workosSync) + .values({ + id: WORKOS_EVENTS_STREAM_ID, + cursor: null, + backfillCompletedAt: at, + updatedAt: at, + }) + .onConflictDoUpdate({ + target: workosSync.id, + set: { backfillCompletedAt: at }, + // The first completion stands, whatever this run's is. + setWhere: isNull(workosSync.backfillCompletedAt), + }) + .returning({ id: workosSync.id }); + return recorded.length > 0; + }), + + organizationBackfilledAt: (organizationId) => + run("organizationBackfilledAt", async () => { + const rows = await db + .select({ backfilledAt: organizations.backfilledAt }) + .from(organizations) + .where(eq(organizations.id, organizationId)); + return rows[0]?.backfilledAt ?? null; + }), + }; +}; diff --git a/apps/cloud/src/auth/workos-mirror.node.test.ts b/apps/cloud/src/auth/workos-mirror.node.test.ts index 61e7357399..25e493385b 100644 --- a/apps/cloud/src/auth/workos-mirror.node.test.ts +++ b/apps/cloud/src/auth/workos-mirror.node.test.ts @@ -6,10 +6,13 @@ // // What this pins: // - an older WorkOS payload never overwrites a newer row (replay-safe) -// - a delete tombstones the row (inactive, stamped with the deletion time): -// a stale OR equal-timestamp upsert cannot resurrect it, a newer one -// reactivates it, and every default read treats the tombstone as no -// membership +// - a delete tombstones the row (inactive, `deleted_at` set) by IDENTITY: +// no payload naming the deleted membership id ever reactivates it, +// however it is stamped — not a stale one, not one stamped after the +// removal — while a replacement under a NEW id takes the row over live; +// every default read treats the tombstone as no membership +// - a membership WorkOS merely deactivated (inactive, no `deleted_at`) +// reactivates under the same id like any other update // - a delete of a membership or user the mirror has not seen yet leaves // the tombstone behind, so the backfill's older payload cannot insert // the row live afterwards @@ -17,25 +20,40 @@ // still carries the id it was replaced from) is recorded all the same, // so a later, newer payload of the deleted id cannot take the row over // - a deleted user takes no membership at all, however the payload is -// stamped: the account tombstone refuses the insert by identity +// stamped: the account tombstone refuses the insert by identity — and a +// membership write racing the deletion waits for its commit and sees +// the tombstone, never inserting a live membership for a deleted user // - a delete with no WorkOS instant keeps the row's own WorkOS stamp, so // a replacement membership WorkOS created meanwhile is not refused, // while the removed membership's own payload still is // - the cursor advances only by compare-and-set (one owner per stream) +// - a backfill scan is applied only if its listing is newer than the one +// already applied to the organization (one owner per listing instant), +// so an older listing cannot insert a membership the newer one lacked; +// a deleted organization takes no scan +// - a membership payload stamped before the organization's last scan is +// refused (a login list fetched before a revocation the scan already +// applied cannot reinstate it); one stamped at or after it is written — +// and a write racing a scan waits for the scan's commit and sees its +// mark, so it cannot insert a membership the scan just proved revoked +// - the events replay boundary is recorded once and never advanced // - `members` searches email AND name case-insensitively, pages stably // - `findByEmail` ignores the casing WorkOS stored // - a membership arriving before its user still holds (FK via ensureAccount) // --------------------------------------------------------------------------- import { describe, expect, it } from "@effect/vitest"; -import { Effect, Layer } from "effect"; +import { eq, sql } from "drizzle-orm"; +import { Context, Deferred, Effect, Fiber, Layer, Option } from "effect"; import { MemberDirectory } from "@executor-js/api/server"; -import { DbService } from "../db/db"; +import { DbService, makeDbLayer } from "../db/db"; +import { accounts, organizations } from "../db/schema"; import { cloudMemberDirectoryLayer } from "./member-directory"; import { UserStoreService } from "./context"; import { WorkOsMirror, type WorkOsMirrorMembership, type WorkOsMirrorUser } from "./workos-mirror"; +import { makeWorkOsMirrorStore } from "./workos-mirror-store"; const DbLive = DbService.Live; const Services = Layer.mergeAll( @@ -44,8 +62,14 @@ const Services = Layer.mergeAll( UserStoreService.Live, ).pipe(Layer.provideMerge(DbLive)); -const run = (body: Effect.Effect) => - Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped)); +const run = ( + body: Effect.Effect, +) => Effect.runPromise(body.pipe(Effect.provide(Services), Effect.scoped)); + +/** The events row is instance-wide: drop it so a test starts as a never-backfilled database. */ +const clearEventsRow = Effect.flatMap(DbService.asEffect(), ({ db }) => + Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)), +); const at = (iso: string) => new Date(iso); const T1 = at("2026-01-01T00:00:00.000Z"); @@ -59,7 +83,9 @@ const freshOrg = () => Effect.gen(function* () { const id = `org_${crypto.randomUUID().replaceAll("-", "")}`; const store = yield* UserStoreService; - yield* store.use("upsertOrganization", (s) => s.upsertOrganization({ id, name: "Mirror Org" })); + yield* store.use("upsertOrganization", (s) => + s.upsertOrganization({ id, name: "Mirror Org", updatedAt: T1 }), + ); return id; }); @@ -168,7 +194,7 @@ describe("WorkOsMirror upserts", () => { expect(result.filled?.name).toBe("Late"); }); - it("tombstones a deleted membership so a stale upsert cannot resurrect it, and a newer one can", async () => { + it("tombstones a deleted membership by identity: no payload of that id resurrects it, a replacement under a new id does", async () => { const result = await run( Effect.gen(function* () { const mirror = yield* WorkOsMirror; @@ -198,6 +224,14 @@ describe("WorkOsMirror upserts", () => { membership(org, id, { id: membershipId, updatedAt: T2 }), ); const afterEqual = yield* directory.membership(id, org, ["inactive"]); + // A role change issued before the removal and delivered after it (a + // stalled login list, a lagging feeder), stamped NEWER than anything + // the row holds — the payload a timestamp guard would let through. + // Same id: the membership is deleted, it never returns. + const newerSameId = yield* mirror.upsertMembership( + membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }), + ); + const afterNewerSameId = yield* directory.membership(id, org, ["inactive"]); // The member re-added in WorkOS: a payload newer than the deletion, // under a new membership id (a deleted id is never reused). const readded = yield* mirror.upsertMembership( @@ -224,6 +258,8 @@ describe("WorkOsMirror upserts", () => { afterStale, equal, afterEqual, + newerSameId, + afterNewerSameId, readded, afterReadd, lateDelete, @@ -245,7 +281,12 @@ describe("WorkOsMirror upserts", () => { expect(result.afterStale?.status).toBe("inactive"); expect(result.equal, "an upsert stamped AT the deletion is refused too").toBe(false); expect(result.afterEqual?.status).toBe("inactive"); - expect(result.readded, "an upsert newer than the deletion reactivates").toBe(true); + expect( + result.newerSameId, + "a payload of the deleted id stamped AFTER the removal is refused: identity, not time", + ).toBe(false); + expect(result.afterNewerSameId).toMatchObject({ status: "inactive", role: "member" }); + expect(result.readded, "a replacement under a new id reactivates").toBe(true); expect(result.afterReadd?.status).toBe("active"); expect(result.lateDelete, "a replayed deletion of the OLD id is refused").toBe(false); expect(result.afterLateDelete?.status).toBe("active"); @@ -447,6 +488,59 @@ describe("WorkOsMirror upserts", () => { expect(result.afterReadd?.status).toBe("active"); }); + it("reactivates a membership WorkOS deactivated under the same id: a deactivation is not a deletion", async () => { + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const directory = yield* MemberDirectory; + const org = yield* freshOrg(); + const id = `user_${crypto.randomUUID()}`; + const membershipId = `om_${id}_${org}`; + yield* mirror.upsertUser(user(id)); + yield* mirror.upsertMembership(membership(org, id, { id: membershipId, updatedAt: T1 })); + // WorkOS deactivates the membership (an `organization_membership.updated` + // with status inactive): the row is inactive but still WorkOS's, with + // no `deleted_at` to protect it. + const deactivated = yield* mirror.upsertMembership( + membership(org, id, { id: membershipId, status: "inactive", updatedAt: T2 }), + ); + const whileInactive = yield* directory.membership(id, org); + // A payload older than the deactivation cannot undo it, nor one + // stamped at the same instant. + const older = yield* mirror.upsertMembership( + membership(org, id, { id: membershipId, updatedAt: T1 }), + ); + const equal = yield* mirror.upsertMembership( + membership(org, id, { id: membershipId, updatedAt: T2 }), + ); + // WorkOS reactivates it, same id, newer stamp: live again. + const reactivated = yield* mirror.upsertMembership( + membership(org, id, { id: membershipId, role: "admin", updatedAt: T3 }), + ); + const afterReactivate = yield* directory.membership(id, org); + return { + membershipId, + deactivated, + whileInactive, + older, + equal, + reactivated, + afterReactivate, + }; + }), + ); + expect(result.deactivated).toBe(true); + expect(result.whileInactive, "an inactive membership reads as no membership").toBeNull(); + expect(result.older, "an older payload cannot undo the deactivation").toBe(false); + expect(result.equal, "nor one stamped at the deactivation").toBe(false); + expect(result.reactivated, "a newer payload under the same id reactivates it").toBe(true); + expect(result.afterReactivate).toMatchObject({ + status: "active", + role: "admin", + membershipId: result.membershipId, + }); + }); + it("tombstones every membership of a deleted user and clears the profile, keeping the account row", async () => { const result = await run( Effect.gen(function* () { @@ -465,11 +559,13 @@ describe("WorkOsMirror upserts", () => { const inB = yield* directory.membership(id, orgB, ["inactive"]); // A stale user payload cannot restore the profile, nor can one // stamped at the deletion itself. No membership of the deleted user - // is written again — not one stamped after the deletion under a new - // id, the payload a timestamp guard would let through: the user is - // gone, and WorkOS never reuses the id. + // is written again — not the deleted id (a deleted membership id + // never returns), and not one stamped after the deletion under a + // NEW id, the payload a timestamp guard would let through: the user + // is gone, and WorkOS never reuses the id. const staleUser = yield* mirror.upsertUser(user(id, { firstName: "Back", updatedAt: T1 })); const equalUser = yield* mirror.upsertUser(user(id, { firstName: "Same", updatedAt: T2 })); + const sameId = yield* mirror.upsertMembership(membership(orgA, id, { updatedAt: T3 })); const rejoined = yield* mirror.upsertMembership( membership(orgA, id, { id: `om_${id}_${orgA}_2`, updatedAt: T3 }), ); @@ -499,6 +595,7 @@ describe("WorkOsMirror upserts", () => { inB, staleUser, equalUser, + sameId, rejoined, afterRejoin, unknown, @@ -518,14 +615,12 @@ describe("WorkOsMirror upserts", () => { }); expect(result.staleUser).toBe(false); expect(result.equalUser, "a profile stamped AT the deletion is refused").toBe(false); + expect(result.sameId, "a deleted membership id never returns").toBe(false); expect( result.rejoined, "a membership of a deleted user is refused however it is stamped: identity, not time", ).toBe(false); - expect(result.afterRejoin).toMatchObject({ - status: "inactive", - name: null, - }); + expect(result.afterRejoin).toMatchObject({ status: "inactive", name: null }); expect(result.unknown, "deleting an unseen user leaves a tombstone").toBe(true); expect(result.unseenProfile, "which the older profile cannot fill").toBe(false); expect( @@ -534,6 +629,150 @@ describe("WorkOsMirror upserts", () => { ).toBe(false); expect(result.unseenRow).toBeNull(); }); + + it("waits for a user deletion holding the account row before judging a membership write against its tombstone", async () => { + // A feeder (a stalled login list, the backfill's older listing) writes a + // membership of a user whose `user.deleted` the reconciler is applying + // at this moment. The deletion locks the account row for the length of + // its transaction; the feeder's write must wait for that commit, see the + // tombstone, and refuse the payload — never insert a live membership + // for a deleted user. The deletion runs on the test's shared connection; + // the feeder runs the same store over a second one, so the two + // transactions are real peers on the server. + const result = await run( + Effect.scoped( + Effect.gen(function* () { + const org = yield* freshOrg(); + const gone = `user_${crypto.randomUUID()}`; + const mirror = yield* WorkOsMirror; + const { db: deleterDb } = yield* DbService; + // The second connection is owned by this test body's scope. + const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db; + const feeder = makeWorkOsMirrorStore(feederDb); + const directory = yield* MemberDirectory; + yield* mirror.upsertUser(user(gone, { firstName: "Gone" })); + + // The deletion's transaction: the account row is locked as + // `deleteUser` locks it, and the transaction is held open until + // `release` — the window a feeder can race. The tombstone itself + // is written after the feeder has started waiting. + const locked = yield* Deferred.make(); + let release: () => void = () => undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + const deletion = yield* Effect.forkChild( + Effect.promise(() => + deleterDb.transaction(async (tx) => { + await tx + .select({ id: accounts.id }) + .from(accounts) + .where(eq(accounts.id, gone)) + .for("no key update"); + await Effect.runPromise(Deferred.succeed(locked, undefined)); + await held; + await makeWorkOsMirrorStore(tx).deleteUser(gone, T2).pipe(Effect.runPromise); + }), + ), + { startImmediately: true }, + ); + yield* Deferred.await(locked); + + // The feeder's write starts while the deletion holds the row. + const written = yield* Deferred.make(); + yield* Effect.forkChild( + feeder + .upsertMembership(membership(org, gone, { updatedAt: T3 })) + .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))), + { startImmediately: true }, + ); + const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis"); + release(); + yield* Fiber.join(deletion); + const afterCommit = yield* Deferred.await(written); + const row = yield* directory.membership(gone, org, ["active", "pending", "inactive"]); + return { beforeCommit, afterCommit, row }; + }), + ), + ); + expect( + Option.isNone(result.beforeCommit), + "the write waits while the deletion holds the account row", + ).toBe(true); + expect( + result.afterCommit, + "once the deletion has committed, its tombstone refuses the payload", + ).toBe(false); + expect(result.row, "so no membership was inserted for the deleted user").toBeNull(); + }); + + it("waits for a membership deletion holding the account row before judging a write of that id against the ledger", async () => { + // Same race for a single membership: a feeder writes membership `om` + // while the reconciler is applying its `organization_membership.deleted`. + // The delete locks the account row for its transaction, so the feeder's + // FOR SHARE read waits, then finds the ledger entry and refuses. Without + // the lock the feeder could pass the ledger check first and insert the + // deleted membership live after the delete committed. + const result = await run( + Effect.scoped( + Effect.gen(function* () { + const org = yield* freshOrg(); + const id = `user_${crypto.randomUUID()}`; + const mirror = yield* WorkOsMirror; + const { db: deleterDb } = yield* DbService; + const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db; + const feeder = makeWorkOsMirrorStore(feederDb); + const directory = yield* MemberDirectory; + yield* mirror.upsertUser(user(id)); + const gone = membership(org, id, { updatedAt: T3 }); + + const locked = yield* Deferred.make(); + let release: () => void = () => undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + const deletion = yield* Effect.forkChild( + Effect.promise(() => + deleterDb.transaction(async (tx) => { + await tx + .select({ id: accounts.id }) + .from(accounts) + .where(eq(accounts.id, id)) + .for("no key update"); + await Effect.runPromise(Deferred.succeed(locked, undefined)); + await held; + await makeWorkOsMirrorStore(tx).deleteMembership(gone, T2).pipe(Effect.runPromise); + }), + ), + { startImmediately: true }, + ); + yield* Deferred.await(locked); + + const written = yield* Deferred.make(); + yield* Effect.forkChild( + feeder + .upsertMembership(gone) + .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))), + { startImmediately: true }, + ); + const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis"); + release(); + yield* Fiber.join(deletion); + const afterCommit = yield* Deferred.await(written); + const row = yield* directory.membership(id, org, ["active", "pending", "inactive"]); + return { beforeCommit, afterCommit, status: row?.status ?? null }; + }), + ), + ); + expect( + Option.isNone(result.beforeCommit), + "the write waits while the deletion holds the account row", + ).toBe(true); + expect(result.afterCommit, "once the deletion has committed, the ledger refuses the id").toBe( + false, + ); + expect(result.status, "and the row is the tombstone, never live").toBe("inactive"); + }); }); describe("WorkOsMirror cursor", () => { @@ -560,6 +799,291 @@ describe("WorkOsMirror cursor", () => { }); }); +describe("WorkOsMirror backfill sync state", () => { + it("records the replay boundary and the backfill completion once each, without touching the cursor", async () => { + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + // The events row is instance-wide (migration 0019 seeds it on the + // empty test database, other tests may have written it): start from + // no row, as a database that has never been backfilled has. + yield* clearEventsRow; + const first = yield* mirror.setReplayBoundary(T2); + const boundary = yield* mirror.replayBoundary(); + const cursorAfterBoundary = yield* mirror.getCursor(); + // A later completed backfill: its boundary is not recorded. + const again = yield* mirror.setReplayBoundary(T3); + const afterAgain = yield* mirror.replayBoundary(); + // Nor once the stream is being followed. + const cursorBefore = yield* mirror.getCursor(); + yield* mirror.setCursor(cursorBefore, "event_boundary"); + const afterCursor = yield* mirror.setReplayBoundary(T1); + const boundaryWithCursor = yield* mirror.replayBoundary(); + const cursor = yield* mirror.getCursor(); + // The completion mark: absent until a run covers every organization, + // then written once, beside the boundary and the cursor. + const notCompleted = yield* mirror.backfillCompletedAt(); + const completed = yield* mirror.markBackfillCompleted(T3); + const completedAgain = yield* mirror.markBackfillCompleted(T4); + const completedAt = yield* mirror.backfillCompletedAt(); + const boundaryAfterCompletion = yield* mirror.replayBoundary(); + const cursorAfterCompletion = yield* mirror.getCursor(); + return { + first, + boundary, + cursorAfterBoundary, + again, + afterAgain, + afterCursor, + boundaryWithCursor, + cursor, + notCompleted, + completed, + completedAgain, + completedAt, + boundaryAfterCompletion, + cursorAfterCompletion, + }; + }), + ); + expect(result.first, "the first boundary is recorded").toBe(true); + expect(result.boundary, "and reads back as written").toEqual(T2); + expect(result.cursorAfterBoundary, "writing the boundary mints no cursor").toBeNull(); + expect(result.again, "a later run's boundary is refused").toBe(false); + expect(result.afterAgain, "the first stands").toEqual(T2); + expect(result.afterCursor).toBe(false); + expect(result.boundaryWithCursor).toEqual(T2); + expect(result.cursor, "and the cursor is untouched").toBe("event_boundary"); + expect(result.notCompleted, "no completion until a run covers every org").toBeNull(); + expect(result.completed, "the first completion is recorded").toBe(true); + expect(result.completedAgain, "a later one is refused").toBe(false); + expect(result.completedAt, "the first stands").toEqual(T3); + expect(result.boundaryAfterCompletion, "the boundary is untouched").toEqual(T2); + expect(result.cursorAfterCompletion, "and so is the cursor").toBe("event_boundary"); + }); + + it("refuses a membership payload stamped before the organization's last scan, and accepts one stamped at or after it", async () => { + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const directory = yield* MemberDirectory; + const org = yield* freshOrg(); + const revoked = `user_${crypto.randomUUID()}`; + const kept = `user_${crypto.randomUUID()}`; + const joined = `user_${crypto.randomUUID()}`; + // A login fetched its membership list at T1, while `revoked` was a + // member, then stalled. WorkOS revoked them, and the backfill scanned + // the org at T2 without them — nothing to tombstone, the row was + // never there. + yield* mirror.applyOrganizationScan({ + organizationId: org, + listedAt: T2, + members: [{ user: user(kept), membership: membership(org, kept, { updatedAt: T1 }) }], + }); + // The stalled login resumes and writes what it holds: refused, the + // revocation predates the scan and nothing would ever undo the row. + const stale = yield* mirror.upsertMembership(membership(org, revoked, { updatedAt: T1 })); + const staleRow = yield* directory.membership(revoked, org, [ + "active", + "pending", + "inactive", + ]); + // The same list's payload for a member the scan kept: refused too — + // it changes nothing, the scan already wrote that state. + const repeated = yield* mirror.upsertMembership(membership(org, kept, { updatedAt: T1 })); + const keptRow = yield* directory.membership(kept, org); + // A membership WorkOS created after the scan (its event, or a login + // after it): stamped past the mark, written. + const later = yield* mirror.upsertMembership(membership(org, joined, { updatedAt: T3 })); + const atMark = yield* mirror.upsertMembership( + membership(org, kept, { role: "admin", updatedAt: T2 }), + ); + const keptAfter = yield* directory.membership(kept, org); + return { stale, staleRow, repeated, keptRow, later, atMark, keptAfter }; + }), + ); + expect(result.stale, "a payload older than the scan is refused").toBe(false); + expect(result.staleRow, "and no row is minted for the revoked member").toBeNull(); + expect(result.repeated, "even for a member the scan kept").toBe(false); + expect(result.keptRow?.status).toBe("active"); + expect(result.later, "a payload newer than the scan is written").toBe(true); + expect(result.atMark, "as is one stamped at the scan's instant").toBe(true); + expect(result.keptAfter?.role).toBe("admin"); + }); + + it("waits for a scan holding the organization row before judging a payload against the scan's mark", async () => { + // A feeder (a login) writes a membership it fetched at T1 while a scan + // listed at T2 — which no longer contains that membership — is being + // applied. The scan claims the organization row for the length of its + // transaction; the feeder's write must wait for that commit, observe + // the mark, and refuse the payload — never insert the membership the + // scan proved revoked. The scan holds the test's shared connection; the + // feeder runs the same store over a second one, so the two transactions + // are real peers on the server. + const result = await run( + Effect.scoped( + Effect.gen(function* () { + const org = yield* freshOrg(); + const revoked = `user_${crypto.randomUUID()}`; + const { db: scanDb } = yield* DbService; + // The second connection is owned by this test body's scope. + const feederDb = Context.get(yield* Layer.build(makeDbLayer()), DbService).db; + const feeder = makeWorkOsMirrorStore(feederDb); + const directory = yield* MemberDirectory; + + // The scan's transaction: its claim (the `backfilled_at` CAS, an + // UPDATE that locks the row) is done, and the transaction is held + // open until `release` — the window a feeder can race. + const claimed = yield* Deferred.make(); + let release: () => void = () => undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + const scan = yield* Effect.forkChild( + Effect.promise(() => + scanDb.transaction(async (tx) => { + await tx + .update(organizations) + .set({ backfilledAt: T2 }) + .where(eq(organizations.id, org)); + await Effect.runPromise(Deferred.succeed(claimed, undefined)); + await held; + }), + ), + { startImmediately: true }, + ); + yield* Deferred.await(claimed); + + // The feeder's write starts while the scan holds the row. + const written = yield* Deferred.make(); + yield* Effect.forkChild( + feeder + .upsertMembership(membership(org, revoked, { updatedAt: T1 })) + .pipe(Effect.flatMap((wrote) => Deferred.succeed(written, wrote))), + { startImmediately: true }, + ); + const beforeCommit = yield* Effect.timeoutOption(Deferred.await(written), "250 millis"); + release(); + yield* Fiber.join(scan); + const afterCommit = yield* Deferred.await(written); + const row = yield* directory.membership(revoked, org, ["active", "pending", "inactive"]); + return { beforeCommit, afterCommit, row }; + }), + ), + ); + expect( + Option.isNone(result.beforeCommit), + "the write waits while the scan holds the organization row", + ).toBe(true); + expect(result.afterCommit, "once the scan has committed, its mark refuses the payload").toBe( + false, + ); + expect(result.row, "so the revoked membership was never inserted").toBeNull(); + }); + + it("applies a scan only when its listing is newer than the one already applied, and never to a deleted or unknown organization", async () => { + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const directory = yield* MemberDirectory; + const store = yield* UserStoreService; + const scanned = yield* freshOrg(); + const untouched = yield* freshOrg(); + const staying = `user_${crypto.randomUUID()}`; + const leaving = `user_${crypto.randomUUID()}`; + const member = (id: string) => ({ + user: user(id), + membership: membership(scanned, id), + }); + + const before = yield* mirror.organizationBackfilledAt(scanned); + // The later listing (T2) no longer contains `leaving`; applied first. + const later = yield* mirror.applyOrganizationScan({ + organizationId: scanned, + listedAt: T2, + members: [member(staying)], + }); + const afterLater = yield* mirror.organizationBackfilledAt(scanned); + // The earlier listing (T1) still contains `leaving`: refused whole. + const earlier = yield* mirror.applyOrganizationScan({ + organizationId: scanned, + listedAt: T1, + members: [member(staying), member(leaving)], + }); + const afterEarlier = yield* mirror.organizationBackfilledAt(scanned); + const leavingRow = yield* directory.membership(leaving, scanned, [ + "active", + "pending", + "inactive", + ]); + // The same instant is not newer either: a replayed listing writes nothing. + const same = yield* mirror.applyOrganizationScan({ + organizationId: scanned, + listedAt: T2, + members: [], + }); + const stayingRow = yield* directory.membership(staying, scanned); + const other = yield* mirror.organizationBackfilledAt(untouched); + const unknown = yield* mirror.applyOrganizationScan({ + organizationId: "org_never_mirrored", + listedAt: T3, + members: [], + }); + yield* store.use("deleteOrganizationCascade", (s) => + s.deleteOrganizationCascade(untouched, T3), + ); + const deleted = yield* mirror.applyOrganizationScan({ + organizationId: untouched, + listedAt: T4, + members: [{ user: user(staying), membership: membership(untouched, staying) }], + }); + const deletedRow = yield* directory.membership(staying, untouched); + return { + before, + later, + afterLater, + earlier, + afterEarlier, + leavingRow, + same, + stayingRow, + other, + unknown, + deleted, + deletedRow, + }; + }), + ); + expect(result.before, "a freshly mirrored organization is unscanned").toBeNull(); + expect(result.later).toEqual( + Option.some({ + usersWritten: 1, + membershipsWritten: 1, + membershipsTombstoned: 0, + }), + ); + expect(result.afterLater, "the scan marks the organization as of its listing").toEqual(T2); + expect(result.earlier, "an older listing is refused whole").toEqual(Option.none()); + expect(result.afterEarlier, "and the mark never moves backwards").toEqual(T2); + expect( + result.leavingRow, + "the membership only the older listing held was never written", + ).toBeNull(); + expect(result.same, "a listing at the recorded instant is refused too").toEqual(Option.none()); + expect(result.stayingRow?.status, "so it tombstones nothing the newer one wrote").toBe( + "active", + ); + expect(result.other, "another organization's mark is its own").toBeNull(); + expect(result.unknown, "an organization the mirror does not hold takes no scan").toEqual( + Option.none(), + ); + expect(result.deleted, "nor does a deleted one, however new the listing").toEqual( + Option.none(), + ); + expect(result.deletedRow).toBeNull(); + }); +}); + describe("cloud MemberDirectory", () => { const seed = (org: string) => Effect.gen(function* () { diff --git a/apps/cloud/src/auth/workos-mirror.ts b/apps/cloud/src/auth/workos-mirror.ts index f3869e430a..6898facdb5 100644 --- a/apps/cloud/src/auth/workos-mirror.ts +++ b/apps/cloud/src/auth/workos-mirror.ts @@ -5,509 +5,43 @@ // `accounts` / `memberships` rows (db/schema.ts) in step with it so the read // side (`auth/member-directory.ts`, the cloud `MemberDirectory`) never has to // ask WorkOS. Three feeders write through it: the login callback (user + -// memberships already in hand), Executor-initiated changes (write-through), -// and the WorkOS Events API reconciler (dashboard-side changes, replayed in -// order from a persisted cursor). +// memberships already in hand), Executor-initiated changes (write-through in +// `auth/handlers.ts` and `account/workos-account-service.ts`), and the WorkOS +// Events API reconciler (dashboard-side changes, replayed in order from a +// persisted cursor). The one-off backfill runs the same store out-of-band. // -// Every write is idempotent and out-of-order safe. Both upserts carry the -// WorkOS `updatedAt` of their payload and refuse to overwrite a row whose -// stored `workos_updated_at` is newer, so a replayed or late-arriving event -// can never regress the mirror. A delete never drops the row: it TOMBSTONES -// it (`status = 'inactive'`, `workos_updated_at` = the deletion time) — and -// INSERTS the tombstone when the row is not there yet — so a feeder that -// fetched the membership before the deletion and writes it after (a login, -// the backfill) is refused by that same guard instead of reinstating access; -// only a payload newer than the deletion (the member re-added in WorkOS) -// reactivates it. At an EQUAL timestamp the tombstone wins: a live row -// accepts a payload stamped the same instant (feeders replay the same -// payload and must converge), a tombstone does not, so a deletion at T is -// never undone by an active payload at T. Every stamp is on WorkOS's clock: -// a deletion replayed from the events stream is stamped with the event's -// time; one Executor made itself — WorkOS answers a delete with no time — -// is stamped with the `updatedAt` WorkOS last reported for the membership, -// or keeps the stamp the row already holds. Never a local clock: read after -// WorkOS answered, it can post-date a replacement membership WorkOS created -// for the same member meanwhile, and a tombstone stamped with it would -// refuse that replacement for good. The row tombstone can only speak for the -// membership id the row happens to hold, so every delete ALSO records the -// deleted WorkOS id in `membership_tombstones`, a ledger keyed by that id -// alone: WorkOS never reuses a deleted `om_…` id, so no membership write -// naming a recorded id is ever current, however it is stamped. That is what -// covers the case the row cannot: membership A replaced by B in WorkOS before -// the mirror saw either, B then deleted — the delete finds a row holding A -// (not B's to tombstone) and would otherwise leave nothing behind, and a -// later scan that still lists B would insert it live, stamped after A. With -// the ledger the delete is recorded whatever the row holds, and the scan's -// payload is refused by identity. A deleted USER is protected by identity -// the same way: `deleteUser` leaves the account row behind as a tombstone -// (profile cleared, stamped with the deletion), and no membership naming -// that account is ever written again, however the payload is stamped — -// WorkOS never reuses a user id, and a membership the mirror had not seen -// has no row of its own for a timestamp guard to refuse the insert against. -// The cursor advances only by compare-and-set, so two reconciler runs cannot -// both own the stream. -// -// Per-request layer shape, like `UserStoreService`: it holds the request's -// postgres socket, so it is rebuilt per request (`RequestScopedServicesLive`) -// and never shared across Workers requests. +// The queries live in `workos-mirror-store.ts`; this file binds them to the +// per-request `DbService`. Per-request layer shape, like `UserStoreService`: +// it holds the request's postgres socket, so it is rebuilt per request +// (`RequestScopedServicesLive`) and never shared across Workers requests. // --------------------------------------------------------------------------- -import { and, eq, isNotNull, isNull, lt, ne, or, sql } from "drizzle-orm"; -import type { AnyPgColumn } from "drizzle-orm/pg-core"; -import { Context, Effect, Layer, Schema } from "effect"; - -import { type MemberStatus } from "@executor-js/api/server"; - -import { accounts, membershipTombstones, memberships, workosSync } from "../db/schema"; -import { DbService, type DrizzleDb } from "../db/db"; -import { - USER_STORE_FAILURE_REASONS, - tryPromiseService, - userStoreReasonFromCause, - withServiceLogging, -} from "./errors"; - -/** A WorkOS user, as the mirror stores it. `updatedAt` is WorkOS's own. */ -export interface WorkOsMirrorUser { - readonly id: string; - readonly email: string; - readonly firstName: string | null; - readonly lastName: string | null; - readonly avatarUrl: string | null; - readonly lastSignInAt: Date | null; - readonly updatedAt: Date; -} - -/** - * A WorkOS organization membership, as the mirror stores it. `id` is the - * WorkOS `om_…`; `accountId` the WorkOS user id; `updatedAt` is WorkOS's own. - * The organization row must already be mirrored (`upsertOrganization`) — a - * membership of an unknown org is a `query` failure, not a silent skip. - */ -export interface WorkOsMirrorMembership { - readonly id: string; - readonly accountId: string; - readonly organizationId: string; - readonly role: string; - readonly status: MemberStatus; - readonly updatedAt: Date; -} - -/** - * What identifies a membership to a delete: the WorkOS `om_…` id AND the - * (account, organization) pair it belongs to. The pair is the row's key, - * and a delete must be able to mint the row as a tombstone when the mirror - * has not seen the membership yet — an id alone cannot. - */ -export type WorkOsMirrorMembershipRef = Pick< - WorkOsMirrorMembership, - "id" | "accountId" | "organizationId" ->; - -/** - * The public failure of every mirror write: which call, and how it failed - * (classified from the driver cause the same way `UserStoreError` is). - */ -export class WorkOsMirrorError extends Schema.TaggedErrorClass()( - "WorkOsMirrorError", - { - operation: Schema.String, - reason: Schema.Literals(USER_STORE_FAILURE_REASONS), - }, - { httpApiStatus: 500 }, -) { - override get message(): string { - return `workos mirror ${this.operation} failed: ${this.reason}`; - } -} - -export interface WorkOsMirrorShape { - /** - * Insert or refresh a user row. `false` when the payload was refused and - * the row left untouched: the stored row is newer than `updatedAt`, or is - * a deletion tombstone stamped at `updatedAt` or later. - */ - readonly upsertUser: (user: WorkOsMirrorUser) => Effect.Effect; - /** - * Insert or refresh a membership row, minting the bare account row first so - * the foreign key holds when the membership arrives before its user. `false` - * when the payload was refused: the stored row is newer than `updatedAt`, - * or is a tombstone stamped at `updatedAt` or later — or the membership id - * is recorded in the deletion ledger (`membership_tombstones`, written by - * `deleteMembership`), whatever the payload is stamped: WorkOS never - * reuses a deleted `om_…` id, so no payload naming a recorded id is - * current — or the account is a deletion tombstone (`deleteUser`), - * whatever the payload is stamped: WorkOS never reuses a user id either, - * so a deleted user has no memberships to mirror. - */ - readonly upsertMembership: ( - membership: WorkOsMirrorMembership, - ) => Effect.Effect; - /** - * Tombstone the membership as deleted: the row stays (or is minted, when - * the mirror has not seen the membership yet), `inactive`, carrying the - * deleted WorkOS id and stamped so that only a payload newer than the - * stamp can reactivate the row. The stamp is on WorkOS's clock, so it - * orders against every payload the mirror is fed: `deletedAt` is an - * instant at or after the membership's last reported state — a deletion - * event's `createdAt`, or the `updatedAt` WorkOS last reported for the - * membership when Executor deletes it (WorkOS answers a delete with no - * time) — or `null` when the caller holds no WorkOS instant at all: the - * row then keeps the stamp it holds, the last state WorkOS reported for - * this membership. Never a local clock: read after WorkOS answered, it - * can post-date a replacement membership WorkOS created for the same - * member meanwhile, and a tombstone stamped with it would refuse that - * replacement for good. Every payload of the deleted membership a feeder - * could have fetched is at or before the stamp (a tombstone wins a tie), - * and every replacement is after it. A row the mirror does not hold and - * no instant to stamp it with takes the current time: there is no WorkOS - * stamp to keep. Matches the row by the membership's IDENTITY, never by - * timestamp: `false` when the row already carries this tombstone or a - * later one (a replayed delete), or holds ANOTHER membership id — the - * member re-added in WorkOS under a new id, which stands whether the - * replacement was mirrored before or after this delete was stamped. In - * EVERY case the deleted id is recorded in the deletion ledger - * (`membership_tombstones`) — a row holding another id is left alone, but - * the id this delete names is still refused to every later write, so a - * membership the mirror never held under its own id (replaced and deleted - * in WorkOS before the mirror saw it) cannot be inserted live by a scan - * that listed it before the deletion. `true` when the delete changed - * anything: the row was tombstoned, or the id was newly recorded. The - * organization row must already be mirrored, as for `upsertMembership`. - */ - readonly deleteMembership: ( - membership: WorkOsMirrorMembershipRef, - deletedAt: Date | null, - ) => Effect.Effect; - /** - * Tombstone a deleted WorkOS user: every membership of the account is - * tombstoned as by `deleteMembership`, and the account row is kept (it - * anchors foreign keys) — or minted, when the mirror has not seen the user - * yet — with its profile cleared and stamped `deletedAt`, so a stale user - * payload cannot restore it. `false` when the account already carries - * this tombstone or a later one (a replayed delete). - */ - readonly deleteUser: ( - accountId: string, - deletedAt: Date, - ) => Effect.Effect; - /** The id of the last WorkOS event applied, or `null` before the first run. */ - readonly getCursor: () => Effect.Effect; - /** - * Compare-and-set the cursor: advance to `next` only if it still reads - * `prev` (`null` = no cursor yet). `false` means another run moved it first - * — the caller must stop, it no longer owns the stream. - */ - readonly setCursor: ( - prev: string | null, - next: string, - ) => Effect.Effect; -} - -// The one events stream the reconciler follows. A row id rather than a -// singleton table so a second stream (another WorkOS environment, a replay) -// can be added without a schema change. -const EVENTS_CURSOR_ID = "events"; - -// A tombstone keeps the row, marks it `inactive`, and moves its timestamp to -// the deletion time — never backwards, so a row that somehow carries a newer -// WorkOS timestamp keeps it and the upsert guard stays at least as strict. -// With no instant at all (`deletedAt` null, see `deleteMembership`) the row -// keeps the stamp it holds, and a row that has none takes the current time -// — the only instant known. (A raw `sql` -// fragment binds the Date without the column's driver mapping, so it is -// passed as ISO text and cast.) -const noEarlierThan = (column: AnyPgColumn, at: Date | null) => - at === null - ? sql`coalesce(${column}, now())` - : sql`greatest(${column}, ${at.toISOString()}::timestamptz)`; - -const tombstone = (deletedAt: Date | null) => ({ - status: "inactive" as const, - workosUpdatedAt: noEarlierThan(memberships.workosUpdatedAt, deletedAt), -}); - -// Which stored membership rows a payload stamped `updatedAt` may overwrite: -// a row with no stamp (predating the mirror), any row stamped earlier, and a -// LIVE row stamped the same instant — feeders replay the same payload and -// must converge, not stall. An `inactive` row stamped the same instant is -// not overwritten: a deletion (or deactivation) at T beats an active payload -// at T, so an equal-timestamp payload can never undo a tombstone. -const membershipAcceptsPayload = (updatedAt: Date) => - or( - isNull(memberships.workosUpdatedAt), - lt(memberships.workosUpdatedAt, updatedAt), - and(eq(memberships.workosUpdatedAt, updatedAt), ne(memberships.status, "inactive")), - ); - -// A deleted user's account row, as `deleteUser` leaves it: the profile is -// cleared (every WorkOS user payload carries an email, so a stamped row with -// none was written by `deleteUser`) and the stamp is the deletion time. A row -// minted bare by `ensureAccount` has no stamp either, and is not a tombstone. -const accountIsTombstone = and(isNull(accounts.email), isNotNull(accounts.workosUpdatedAt)); - -// Whether membership `id` is in the deletion ledger: a WorkOS id a delete -// has named, which never returns. Judged by identity alone — the ledger -// records WHEN for the record, not for ordering. -const membershipIdDeleted = async (db: DrizzleDb, id: string): Promise => { - const rows = await db - .select({ membershipId: membershipTombstones.membershipId }) - .from(membershipTombstones) - .where(eq(membershipTombstones.membershipId, id)); - return rows.length > 0; -}; - -// The same rule as for a membership row, for an account row. A tombstone -// takes no payload stamped at or before the deletion; a bare row takes any. -const accountAcceptsPayload = (updatedAt: Date) => - or( - isNull(accounts.workosUpdatedAt), - lt(accounts.workosUpdatedAt, updatedAt), - and(eq(accounts.workosUpdatedAt, updatedAt), isNotNull(accounts.email)), - ); - -// A delete is applied unless the row already carries this tombstone or a -// later one: a replayed deletion changes nothing and reports so. With no -// instant given, any tombstone the row carries is this one or later. -const notTombstonedSince = (deletedAt: Date | null) => - or( - ne(memberships.status, "inactive"), - isNull(memberships.workosUpdatedAt), - deletedAt === null ? undefined : lt(memberships.workosUpdatedAt, deletedAt), - ); - -// Which (account, organization) row a delete of membership `id` at -// `deletedAt` may tombstone: the row carrying THIS id — whatever its stamp, -// a deleted id is never reused — or a row with no id at all (written before -// the mirror recorded WorkOS ids; the delete fills the id in). Never a row -// under ANOTHER id: that is a different membership of the same account and -// organization, the member re-added in WorkOS after this one was removed, -// and it stands however the two are stamped. Its stamp cannot be trusted to -// order them: a removal Executor makes is stamped by a local clock, and one -// that reads the clock after WorkOS answered can post-date a replacement -// WorkOS created while the request was in flight. Identity orders them, -// timestamps do not. A row already carrying this tombstone or a later one is -// left alone (`notTombstonedSince`) so a replayed delete reports `false`. -const membershipDeletableBy = (id: string, deletedAt: Date | null) => - and( - or(isNull(memberships.membershipId), eq(memberships.membershipId, id)), - notTombstonedSince(deletedAt), - ); - -const makeService = (db: DrizzleDb): WorkOsMirrorShape => { - const run = (op: string, fn: () => Promise) => - withServiceLogging( - `workos_mirror.${op}`, - (failure) => - new WorkOsMirrorError({ - operation: op, - reason: userStoreReasonFromCause(failure), - }), - tryPromiseService(fn), - ); - - // Over `db` or a transaction handle (drizzle's is a `PgDatabase` too). - const ensureAccount = (on: DrizzleDb, id: string) => - on.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id }); - - return { - upsertUser: (user) => - run("upsertUser", async () => { - const written = await db - .insert(accounts) - .values({ - id: user.id, - email: user.email, - firstName: user.firstName, - lastName: user.lastName, - avatarUrl: user.avatarUrl, - lastSignInAt: user.lastSignInAt, - workosUpdatedAt: user.updatedAt, - }) - .onConflictDoUpdate({ - target: accounts.id, - set: { - email: user.email, - firstName: user.firstName, - lastName: user.lastName, - avatarUrl: user.avatarUrl, - lastSignInAt: user.lastSignInAt, - workosUpdatedAt: user.updatedAt, - }, - setWhere: accountAcceptsPayload(user.updatedAt), - }) - .returning({ id: accounts.id }); - return written.length > 0; - }), - - upsertMembership: (membership) => - run("upsertMembership", async () => { - await ensureAccount(db, membership.accountId); - // Never for a DELETED user. The row guard below orders a payload - // against the membership row it would overwrite; a membership the - // mirror has not seen yet has no row, so the guard cannot refuse - // the INSERT — and a feeder that fetched the membership before the - // user was deleted and writes it after (a stalled login list, the - // backfill's older listing) would insert it live. The account - // tombstone is the one row a deleted user always leaves behind, so - // it is consulted first, by identity: WorkOS never reuses a user - // id, so no payload naming a tombstoned account is ever current. - const deleted = await db - .select({ id: accounts.id }) - .from(accounts) - .where(and(eq(accounts.id, membership.accountId), accountIsTombstone)); - if (deleted.length > 0) return false; - // Never under a DELETED membership id. The row guard below can only - // refuse against the id the row holds; a delete of THIS id that - // found the row under another id (see the header) left only the - // ledger entry behind, and that is what refuses the payload here. - if (await membershipIdDeleted(db, membership.id)) return false; - const written = await db - .insert(memberships) - .values({ - accountId: membership.accountId, - organizationId: membership.organizationId, - membershipId: membership.id, - role: membership.role, - status: membership.status, - workosUpdatedAt: membership.updatedAt, - }) - .onConflictDoUpdate({ - target: [memberships.accountId, memberships.organizationId], - set: { - membershipId: membership.id, - role: membership.role, - status: membership.status, - workosUpdatedAt: membership.updatedAt, - }, - setWhere: membershipAcceptsPayload(membership.updatedAt), - }) - .returning({ accountId: memberships.accountId }); - return written.length > 0; - }), - - // An upsert, like the membership write it guards against: a delete the - // mirror sees before the membership itself (the reconciler ahead of the - // backfill) must leave the tombstone behind, or the later, older payload - // would insert the row live. - deleteMembership: (membership, deletedAt) => - run("deleteMembership", () => - db.transaction(async (tx) => { - await ensureAccount(tx, membership.accountId); - // The ledger entry FIRST, whatever the row holds: this is the one - // record of the deletion that does not depend on the row carrying - // the deleted id. A replayed delete finds it there (`DO NOTHING`) - // and records nothing new. - const recorded = await tx - .insert(membershipTombstones) - .values({ - membershipId: membership.id, - accountId: membership.accountId, - organizationId: membership.organizationId, - ...(deletedAt === null ? {} : { deletedAt }), - }) - .onConflictDoNothing({ target: membershipTombstones.membershipId }) - .returning({ membershipId: membershipTombstones.membershipId }); - const tombstoned = await tx - .insert(memberships) - .values({ - accountId: membership.accountId, - organizationId: membership.organizationId, - membershipId: membership.id, - status: "inactive", - workosUpdatedAt: deletedAt ?? new Date(), - }) - .onConflictDoUpdate({ - target: [memberships.accountId, memberships.organizationId], - set: { membershipId: membership.id, ...tombstone(deletedAt) }, - setWhere: membershipDeletableBy(membership.id, deletedAt), - }) - .returning({ accountId: memberships.accountId }); - return recorded.length > 0 || tombstoned.length > 0; - }), - ), - - deleteUser: (accountId, deletedAt) => - run("deleteUser", async () => { - // The account tombstone FIRST, then the memberships. A membership - // write checks the account tombstone before it inserts - // (`upsertMembership`), so a write racing this delete either sees - // the tombstone and refuses, or has landed before it and is caught - // by the membership tombstoning below. In the other order a write - // between the two statements would slip through live. - // - // The account tombstone: profile cleared, stamped with the deletion. - // Minted when absent, for the same reason as the membership one. - const cleared = await db - .insert(accounts) - .values({ id: accountId, workosUpdatedAt: deletedAt }) - .onConflictDoUpdate({ - target: accounts.id, - set: { - email: null, - firstName: null, - lastName: null, - avatarUrl: null, - workosUpdatedAt: noEarlierThan(accounts.workosUpdatedAt, deletedAt), - }, - // Applied unless the row already carries this tombstone or a - // later one (a replayed delete). - setWhere: or( - isNotNull(accounts.email), - isNull(accounts.workosUpdatedAt), - lt(accounts.workosUpdatedAt, deletedAt), - ), - }) - .returning({ id: accounts.id }); - await db - .update(memberships) - .set(tombstone(deletedAt)) - .where(and(eq(memberships.accountId, accountId), notTombstonedSince(deletedAt))); - return cleared.length > 0; - }), - - getCursor: () => - run("getCursor", async () => { - const rows = await db - .select({ cursor: workosSync.cursor }) - .from(workosSync) - .where(eq(workosSync.id, EVENTS_CURSOR_ID)); - // No row yet is the same state as a row with no cursor: nothing applied. - return rows[0]?.cursor ?? null; - }), - - setCursor: (prev, next) => - run("setCursor", async () => { - const now = new Date(); - if (prev === null) { - // First advance: mint the row, or claim an existing row that still - // has no cursor. A row that already carries one belongs to another - // run and is left alone. - const written = await db - .insert(workosSync) - .values({ id: EVENTS_CURSOR_ID, cursor: next, updatedAt: now }) - .onConflictDoUpdate({ - target: workosSync.id, - set: { cursor: next, updatedAt: now }, - setWhere: isNull(workosSync.cursor), - }) - .returning({ id: workosSync.id }); - return written.length > 0; - } - const written = await db - .update(workosSync) - .set({ cursor: next, updatedAt: now }) - .where(and(eq(workosSync.id, EVENTS_CURSOR_ID), eq(workosSync.cursor, prev))) - .returning({ id: workosSync.id }); - return written.length > 0; - }), - }; -}; +import { Context, Effect, Layer } from "effect"; + +import { DbService } from "../db/db"; +import { makeWorkOsMirrorStore, type WorkOsMirrorShape } from "./workos-mirror-store"; + +export { WorkOsMirrorError } from "./errors"; +export { + mirrorMembershipFromWorkOs, + mirrorUserFromWorkOs, + type WorkOsMembershipPayload, + type WorkOsMirrorMembership, + type WorkOsMirrorMembershipRef, + type WorkOsMirrorShape, + type WorkOsMirrorUser, + type WorkOsOrganizationScan, + type WorkOsOrganizationScanWrites, + type WorkOsScannedMember, + type WorkOsUserPayload, +} from "./workos-mirror-store"; export class WorkOsMirror extends Context.Service()( "@executor-js/cloud/WorkOsMirror", ) { - static Live = Layer.effect(this)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db))); + static Live = Layer.effect(this)( + Effect.map(DbService.asEffect(), ({ db }) => makeWorkOsMirrorStore(db)), + ); } /** @@ -516,4 +50,6 @@ export class WorkOsMirror extends Context.Service => - Layer.effect(WorkOsMirror)(Effect.map(DbService.asEffect(), ({ db }) => makeService(db))); + Layer.effect(WorkOsMirror)( + Effect.map(DbService.asEffect(), ({ db }) => makeWorkOsMirrorStore(db)), + ); diff --git a/apps/cloud/src/db/db.test.ts b/apps/cloud/src/db/db.test.ts index 889daab6dc..708ea548f0 100644 --- a/apps/cloud/src/db/db.test.ts +++ b/apps/cloud/src/db/db.test.ts @@ -94,7 +94,11 @@ describe("DbService", () => { Effect.gen(function* () { const { db } = yield* DbService; yield* Effect.promise(() => - makeUserStore(db).upsertOrganization({ id: organizationId, name: "Acme" }), + makeUserStore(db).upsertOrganization({ + id: organizationId, + name: "Acme", + updatedAt: new Date(), + }), ); }), ), @@ -121,16 +125,24 @@ describe("DbService", () => { }); describe("upsertOrganization · slug is minted at insert", () => { - const upsert = (org: { id: string; name: string }) => + const upsert = (org: { id: string; name: string; updatedAt?: Date }) => program( Effect.gen(function* () { const { db } = yield* DbService; - return yield* Effect.promise(() => makeUserStore(db).upsertOrganization(org)); + return yield* Effect.promise(() => + makeUserStore(db).upsertOrganization({ + updatedAt: new Date(), + ...org, + }), + ); }), ); it("mints a valid slug on insert", async () => { - const org = await upsert({ id: `org_${crypto.randomUUID()}`, name: "Slug Mint Co" }); + const org = await upsert({ + id: `org_${crypto.randomUUID()}`, + name: "Slug Mint Co", + }); expect(org.slug, "a new org row is born with a slug").toBeTruthy(); expect(isValidOrgSlug(org.slug), "the minted slug fits the URL grammar").toBe(true); }); @@ -143,6 +155,32 @@ describe("upsertOrganization · slug is minted at insert", () => { expect(renamed.name, "the name is refreshed on conflict").toBe("Renamed Org"); }); + it("refuses a name stamped earlier than the one it holds, and never renames a deleted org", async () => { + const id = `org_${crypto.randomUUID()}`; + const t1 = new Date("2026-01-01T00:00:00.000Z"); + const t2 = new Date("2026-01-02T00:00:00.000Z"); + const t3 = new Date("2026-01-03T00:00:00.000Z"); + await upsert({ id, name: "Original Name", updatedAt: t2 }); + // A payload fetched before the rename landed (a login that stalled). + const stale = await upsert({ id, name: "Stale Name", updatedAt: t1 }); + expect(stale.name, "an older name never reverts a newer one").toBe("Original Name"); + const replay = await upsert({ id, name: "Replayed Name", updatedAt: t2 }); + expect(replay.name, "the same instant is accepted, so replays converge").toBe("Replayed Name"); + await program( + Effect.gen(function* () { + const { db } = yield* DbService; + yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(id, t3)); + }), + ); + const afterDelete = await upsert({ + id, + name: "Resurrected Name", + updatedAt: t3, + }); + expect(afterDelete.deletedAt, "a deleted org stays deleted").toEqual(t3); + expect(afterDelete.name, "and keeps its last name").toBe("Replayed Name"); + }); + it("discriminates same-name collisions into distinct slugs", async () => { // Same name → same slug base; the second insert collides on the unique // index and gets a discriminated slug. diff --git a/apps/cloud/src/db/org-deletion.test.ts b/apps/cloud/src/db/org-deletion.test.ts index 979f0b24b3..a7268e2764 100644 --- a/apps/cloud/src/db/org-deletion.test.ts +++ b/apps/cloud/src/db/org-deletion.test.ts @@ -7,7 +7,8 @@ // two orgs across every tenant table + blob namespace, purges one, and asserts: // - every executor tenant table row for the target org is gone // - org- and user-scoped secret blobs for the target org are gone -// - the identity row is gone and its memberships cascade with it +// - the org's memberships are gone; the identity row stays as a tombstone +// marked deleted (so a delayed feeder cannot re-mint the org live) // - a second org's data is completely untouched // - the blob prefix match escapes LIKE wildcards (a `_` in the org id must // not widen the match to a look-alike namespace) @@ -196,11 +197,12 @@ const NOT_ORG_OWNED: Record = { blob: "org-scoped by namespace prefix, purged via the LIKE match", // Instance-wide, not owned by any org. private_executor_cloud_settings: "singleton instance settings, not org-scoped", - // Identity mirror. `organizations` is deleted directly and `memberships` - // cascades from its FK; `accounts` deliberately outlives the org. - organizations: "the identity row itself, deleted directly", - memberships: "cascades from the organizations FK", - membership_tombstones: "cascades from the organizations FK", + // Identity mirror. `organizations` is kept as a tombstone marked deleted, + // `memberships` are deleted by organization id; `accounts` deliberately + // outlives the org. + organizations: "the identity row itself, kept as a tombstone marked deleted", + memberships: "deleted by organization id", + membership_tombstones: "deleted by organization id", accounts: "shared across orgs — deliberately survives", workos_sync: "the WorkOS Events API cursor, instance-wide and not org-scoped", }; @@ -231,14 +233,24 @@ describe("purgeOrganizationData", () => { // A look-alike blob that only an UNescaped `_` wildcard would match: // `o:/…` with the underscore replaced by another char. const trapNs = `o:${orgA.replace("_", "X")}/plugin`; + const now = new Date(); + const deletedAt = new Date("2026-01-02T00:00:00.000Z"); await program( Effect.gen(function* () { const { db } = yield* DbService; yield* Effect.promise(async () => { const store = makeUserStore(db); - await store.upsertOrganization({ id: orgA, name: "Delete Me" }); - await store.upsertOrganization({ id: orgB, name: "Keep Me" }); + await store.upsertOrganization({ + id: orgA, + name: "Delete Me", + updatedAt: now, + }); + await store.upsertOrganization({ + id: orgB, + name: "Keep Me", + updatedAt: now, + }); await store.ensureAccount(accountId); await db.insert(memberships).values({ accountId, organizationId: orgA }); await db.insert(memberships).values({ accountId, organizationId: orgB }); @@ -257,7 +269,7 @@ describe("purgeOrganizationData", () => { await program( Effect.gen(function* () { const { db } = yield* DbService; - yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(orgA)); + yield* Effect.promise(() => makeUserStore(db).deleteOrganizationCascade(orgA, deletedAt)); }), ); @@ -267,10 +279,13 @@ describe("purgeOrganizationData", () => { yield* Effect.promise(async () => { const store = makeUserStore(db); - // Target org: every tenant row + blob gone, identity gone, membership - // cascaded, but the shared account survives (it may join other orgs). + // Target org: every tenant row + blob gone, memberships gone, the + // identity row kept as a tombstone, and the shared account survives + // (it may join other orgs). expect(await countTenantRows(db, orgA)).toBe(0); - expect(await store.getOrganization(orgA)).toBeNull(); + const tombstone = await store.getOrganization(orgA); + expect(tombstone?.deletedAt, "the org row stays, marked deleted").toEqual(deletedAt); + expect(tombstone?.name, "as it was").toBe("Delete Me"); const orgAMemberships = await db .select() .from(memberships) @@ -285,7 +300,7 @@ describe("purgeOrganizationData", () => { // Second org: fully intact. expect(await countTenantRows(db, orgB)).toBeGreaterThan(0); - expect(await store.getOrganization(orgB)).not.toBeNull(); + expect((await store.getOrganization(orgB))?.deletedAt).toBeNull(); const orgBMemberships = await db .select() .from(memberships) diff --git a/apps/cloud/src/db/org-deletion.ts b/apps/cloud/src/db/org-deletion.ts index b2a922a3fd..abcff12f3b 100644 --- a/apps/cloud/src/db/org-deletion.ts +++ b/apps/cloud/src/db/org-deletion.ts @@ -10,11 +10,21 @@ // // External side effects (the WorkOS org, the Autumn customer) are NOT touched // here — the caller (auth handler) sequences those around this purge. +// +// The `organizations` row itself is NOT deleted: it stays as a TOMBSTONE, +// marked `deleted_at`, with its memberships removed. The membership mirror's +// feeders write whatever WorkOS payload they hold — a login that fetched its +// membership list before the deletion can write it after this purge — and +// the tombstone is what makes those writes refuse: `upsertOrganization` +// never re-mints or renames a marked organization, and the mirror never +// inserts a membership of one. Without it the login would insert a fresh, +// live organization row plus an active membership, and the deleted +// organization would authorize again. import { eq, or, sql } from "drizzle-orm"; import type { DrizzleDb } from "./db"; -import { organizations } from "./schema"; +import { memberships, organizations } from "./schema"; import { artifact, blob, @@ -36,10 +46,16 @@ const escapeLike = (value: string): string => value.replace(/[\\%_]/g, "\\$&"); /** * Delete all rows owned by `organizationId`: every executor tenant table, the - * org's secret blobs (org- and user-scoped), and the identity mirror row (which - * cascades to local `memberships`). Idempotent — a second run deletes nothing. + * org's secret blobs (org- and user-scoped), and its local `memberships` — + * and mark the identity row deleted as of `deletedAt` (an earlier mark + * stands), keeping it as a tombstone. Idempotent — a second run deletes + * nothing and keeps the first mark. */ -export const purgeOrganizationData = (db: DrizzleDb, organizationId: string): Promise => +export const purgeOrganizationData = ( + db: DrizzleDb, + organizationId: string, + deletedAt: Date, +): Promise => db.transaction(async (tx) => { // Executor tenant tables — every row is scoped by `tenant = organizationId`. await tx.delete(tool).where(eq(tool.tenant, organizationId)); @@ -66,7 +82,14 @@ export const purgeOrganizationData = (db: DrizzleDb, organizationId: string): Pr ), ); - // Identity mirror — FK `ON DELETE CASCADE` removes local memberships too. - // `accounts` are intentionally left: a user may belong to other orgs. - await tx.delete(organizations).where(eq(organizations.id, organizationId)); + // Identity mirror: the memberships go, the organization row stays as a + // tombstone (see the header). `accounts` are intentionally left: a user + // may belong to other orgs. + await tx.delete(memberships).where(eq(memberships.organizationId, organizationId)); + await tx + .update(organizations) + .set({ + deletedAt: sql`coalesce(${organizations.deletedAt}, ${deletedAt.toISOString()}::timestamptz)`, + }) + .where(eq(organizations.id, organizationId)); }); diff --git a/apps/cloud/src/db/schema.ts b/apps/cloud/src/db/schema.ts index ef18c48248..9bbd73290b 100644 --- a/apps/cloud/src/db/schema.ts +++ b/apps/cloud/src/db/schema.ts @@ -9,7 +9,9 @@ // - `organizations` — billing entity, scoping root for all domain data // - `memberships` — which accounts belong to which organizations, with // the WorkOS role and status -// - `workos_sync` — the WorkOS Events API cursor the reconciler resumes from +// - `workos_sync` — the WorkOS Events API cursor the reconciler resumes +// from, and the replay boundary the one-off backfill +// records // // The mirror is fed by login (the callback has the user + memberships in // hand), write-through on every Executor-initiated change, and the WorkOS @@ -65,6 +67,41 @@ export const organizations = pgTable( id: text("id").primaryKey(), name: text("name").notNull(), slug: text("slug").notNull(), + /** + * When this organization's membership list was last FULLY scanned from + * WorkOS (the one-off backfill, or the on-demand scan a seat count + * triggers), or null if it never was. Until then the mirror may hold only + * the members login and write-through happened to record, so a count read + * from it is partial; every seat gate checks this mark first. Per + * organization, never database-wide: an org mirrored lazily after a + * backfill ran starts unmarked and is scanned on its first count. The + * mark also orders membership writes: a payload stamped before it is + * refused, since the scan was the full listing at that instant and a + * membership it did not contain was revoked before it — before the + * events replay boundary, so nothing would tombstone it again. + */ + backfilledAt: timestamp("backfilled_at", { withTimezone: true }), + /** + * When this organization was deleted, or null while it is live. Set by + * cloud's own deletion flow and by the `organization.deleted` event, and + * KEPT by the local purge (`db/org-deletion.ts`), which removes the + * organization's memberships and tenant data but leaves this row as a + * tombstone: a feeder that fetched a membership before the deletion and + * writes it after (a login that stalled across the purge) finds the + * tombstone and does not re-mint the organization live. A marked + * organization is never renamed and authorizes nobody. + */ + deletedAt: timestamp("deleted_at", { withTimezone: true }), + /** + * The instant the stored `name` is known to have been the organization's + * name in WorkOS: the WorkOS `updatedAt` of the organization payload that + * wrote it, or — for a name learned from a membership list at sign-in, + * which carries no organization timestamp — the instant that list was + * fetched. A name write stamped earlier than this is refused + * (`upsertOrganization`), so a sign-in whose list predates a rename cannot + * revert it. Null only on rows written before the stamp existed. + */ + workosUpdatedAt: timestamp("workos_updated_at", { withTimezone: true }), createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), }, (t) => ({ @@ -157,13 +194,42 @@ export const membershipTombstones = pgTable( ); /** - * The WorkOS Events API cursor. One row per stream (`id` names the stream; - * the reconciler uses `"events"`), holding the id of the last event applied. - * Advanced only by compare-and-set, so two concurrent reconciler runs cannot - * both believe they own the stream: the loser's CAS fails and it stops. + * The WorkOS Events API sync state. One row per stream (`id` names the + * stream; the reconciler uses `"events"`), holding the id of the last event + * applied. Advanced only by compare-and-set, so two concurrent reconciler + * runs cannot both believe they own the stream: the loser's CAS fails and it + * stops. + * + * `range_start` on the `"events"` row is the REPLAY BOUNDARY: the instant the + * FIRST completed one-off backfill (`scripts/backfill-workos-mirror.ts`) began + * reading WorkOS. Everything before it is covered by that backfill; the + * reconciler's first run (no cursor yet) reads the events stream from here, + * so a revocation between the backfill and the first run is never skipped. + * Written once: a backfill that fails part-way records nothing, and a later + * completed one keeps it, because the backfill does not refresh everything + * the events stream carries (organization renames, deleted users' + * profiles) — those between two runs are replayed from the first boundary. + * Without a cursor or a boundary the reconciler does not guess; it waits for + * the backfill. + * + * `backfill_completed_at` is when a backfill run first wrote EVERY live + * organization (`scripts/backfill-workos-mirror.ts` completing, or refusing + * an organization only because a later listing was already applied). Until + * it is set, the mirror may lack members who have not signed in since it + * shipped, so a membership check read from it would deny them: it is the + * first half of the mirror-readiness mark the authorization path consults + * before trusting the mirror over WorkOS. Write-once — a later completed run + * keeps the first instant, so readiness never flips back. Per-organization + * completeness for the seat gates is tracked separately + * (`organizations.backfilled_at`). + * + * Migration 0019 seeds the boundary and the completion mark on a database + * with no organizations, where there is nothing to backfill. */ export const workosSync = pgTable("workos_sync", { id: text("id").primaryKey(), cursor: text("cursor"), + rangeStart: timestamp("range_start", { withTimezone: true }), + backfillCompletedAt: timestamp("backfill_completed_at", { withTimezone: true }), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }); diff --git a/apps/cloud/src/extensions/billing/route.node.test.ts b/apps/cloud/src/extensions/billing/route.node.test.ts index b25ec53124..51a182b180 100644 --- a/apps/cloud/src/extensions/billing/route.node.test.ts +++ b/apps/cloud/src/extensions/billing/route.node.test.ts @@ -55,18 +55,27 @@ const stubUsers = Layer.succeed(UserStoreService)({ upsertOrganization: async (org: { id: string; name: string }) => ({ ...org, slug: org.id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganization: async (id: string) => ({ id, name: `Org ${id}`, slug: id, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), getOrganizationBySlug: async (slug: string) => ({ id: slug === URL_SLUG ? URL_ORG : "org_outsider", name: `Org ${slug}`, slug, + backfilledAt: null, + deletedAt: null, + workosUpdatedAt: null, createdAt, }), deleteOrganizationCascade: async () => {}, diff --git a/apps/cloud/src/extensions/routes.ts b/apps/cloud/src/extensions/routes.ts index 4c04bd02cd..e362b62d5a 100644 --- a/apps/cloud/src/extensions/routes.ts +++ b/apps/cloud/src/extensions/routes.ts @@ -28,6 +28,7 @@ import { AccountApi, AdminUsersApi } from "@executor-js/api"; import { requestScopedMiddleware } from "@executor-js/api/server"; import { UserStoreService } from "../auth/context"; +import { WorkOsMirror } from "../auth/workos-mirror"; import { CloudAuthPublicHandlers, CloudSessionAuthHandlers, @@ -72,7 +73,9 @@ const spec = OpenApi.fromApi(CloudOpenApi); * read it — the few app-only billing touchpoints. It is NOT on the neutral boot * core. */ -export const makeCloudExtensionRoutes = (rsLive: Layer.Layer) => { +export const makeCloudExtensionRoutes = ( + rsLive: Layer.Layer, +) => { // Session routes (login / callback / me / switch-org / …). Handlers yield // `UserStoreService` directly; the per-request DB combine keeps the postgres // socket request-scoped.