diff --git a/.changeset/member-directory-reconciler.md b/.changeset/member-directory-reconciler.md new file mode 100644 index 000000000..b73bbf878 --- /dev/null +++ b/.changeset/member-directory-reconciler.md @@ -0,0 +1,7 @@ +--- +"@executor-js/cloud": patch +--- + +The cloud membership mirror is now reconciled from the WorkOS Events API: an every-minute cron replays user, organization-membership, and organization events from a persisted cursor, so changes made in the WorkOS dashboard (a removed member, a role edit, a profile update) reach the mirror without anyone signing in. A signed webhook at `/api/webhooks/workos` pokes the same reconciler so those changes land in seconds, and `bun run --cwd apps/cloud db:drain-workos-events:prod` runs the same replay out-of-band until the stream is drained. + +**Ops steps (cloud):** set the webhook signing secret with `wrangler secret put WORKOS_WEBHOOK_SECRET`, then register `https://executor.sh/api/webhooks/workos` as a webhook endpoint in the WorkOS dashboard for the `user.*`, `organization_membership.*`, `organization.updated`, and `organization.deleted` events. Until the secret is set the route answers 503 and the cron alone keeps the mirror current. diff --git a/apps/cloud/drizzle/0020_workos_sync_drained_at.sql b/apps/cloud/drizzle/0020_workos_sync_drained_at.sql new file mode 100644 index 000000000..f1e1b34c4 --- /dev/null +++ b/apps/cloud/drizzle/0020_workos_sync_drained_at.sql @@ -0,0 +1 @@ +ALTER TABLE "workos_sync" ADD COLUMN "drained_at" timestamp with time zone; \ No newline at end of file diff --git a/apps/cloud/drizzle/meta/0020_snapshot.json b/apps/cloud/drizzle/meta/0020_snapshot.json new file mode 100644 index 000000000..886f0a784 --- /dev/null +++ b/apps/cloud/drizzle/meta/0020_snapshot.json @@ -0,0 +1,1760 @@ +{ + "id": "88f2845b-be28-4ad3-92b2-2cac819478e5", + "prevId": "26e5a445-9146-40bb-afaf-0f26bfb00818", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sign_in_at": { + "name": "last_sign_in_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_email_lower_idx": { + "name": "accounts_email_lower_idx", + "columns": [ + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.membership_tombstones": { + "name": "membership_tombstones", + "schema": "", + "columns": { + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "membership_tombstones_organization_id_idx": { + "name": "membership_tombstones_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "membership_tombstones_account_id_accounts_id_fk": { + "name": "membership_tombstones_account_id_accounts_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "membership_tombstones_organization_id_organizations_id_fk": { + "name": "membership_tombstones_organization_id_organizations_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_membership_id_unique": { + "name": "memberships_membership_id_unique", + "columns": [ + { + "expression": "membership_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_organization_id_idx": { + "name": "memberships_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backfilled_at": { + "name": "backfilled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workos_sync": { + "name": "workos_sync", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "range_start": { + "name": "range_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_completed_at": { + "name": "backfill_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "drained_at": { + "name": "drained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index b629ea283..73842e4d5 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -141,6 +141,13 @@ "when": 1789571259533, "tag": "0019_workos_mirror_sync_state", "breakpoints": true + }, + { + "idx": 20, + "version": "7", + "when": 1789575639971, + "tag": "0020_workos_sync_drained_at", + "breakpoints": true } ] } diff --git a/apps/cloud/package.json b/apps/cloud/package.json index 0c9303168..328d096b5 100644 --- a/apps/cloud/package.json +++ b/apps/cloud/package.json @@ -34,6 +34,8 @@ "db:backfill-subjects:dev": "op run --env-file=.env.op -- bun run scripts/backfill-subjects.ts", "db:backfill-workos-mirror:prod": "op run --env-file=.env.production -- bun run scripts/backfill-workos-mirror.ts", "db:backfill-workos-mirror:dev": "op run --env-file=.env.op -- bun run scripts/backfill-workos-mirror.ts", + "db:drain-workos-events:prod": "op run --env-file=.env.production -- bun run scripts/drain-workos-events.ts", + "db:drain-workos-events:dev": "op run --env-file=.env.op -- bun run scripts/drain-workos-events.ts", "routes:gen": "bun scripts/gen-routes.ts", "vendor-wasm": "bun run scripts/vendor-quickjs-wasm.ts" }, diff --git a/apps/cloud/scripts/backfill-workos-mirror.ts b/apps/cloud/scripts/backfill-workos-mirror.ts index 712138585..945de792b 100644 --- a/apps/cloud/scripts/backfill-workos-mirror.ts +++ b/apps/cloud/scripts/backfill-workos-mirror.ts @@ -24,13 +24,13 @@ // // DEPLOY ORDER: run this against production BEFORE deploying the builds that // reconcile from the Events API and read seat counts from the mirror, so no -// request pays for an on-demand scan. The FIRST completed run records the -// events replay boundary (the reconciler's first run reads from it; without -// one it waits); later runs keep it, since only the events stream covers the -// org renames and user deletions between two runs. A run that fails part-way -// keeps the marks of the orgs it finished, records no boundary, and is safe -// to repeat. Verify the printed membership count against the WorkOS -// dashboard. +// request pays for an on-demand scan. The FIRST run records the events +// replay boundary BEFORE it lists anything (the reconciler's first run reads +// from it; without one it waits); later runs — a retry included — keep it, +// since only the events stream covers the org renames and user deletions +// after that instant. A run that fails part-way keeps the marks of the orgs +// it finished and the boundary it recorded, and is safe to repeat. Verify +// the printed membership count against the WorkOS dashboard. // --------------------------------------------------------------------------- import { asc, isNull } from "drizzle-orm"; diff --git a/apps/cloud/scripts/drain-workos-events.ts b/apps/cloud/scripts/drain-workos-events.ts new file mode 100644 index 000000000..b5cfc6f3d --- /dev/null +++ b/apps/cloud/scripts/drain-workos-events.ts @@ -0,0 +1,128 @@ +// --------------------------------------------------------------------------- +// Out-of-band reconciler run: replay the WorkOS Events API into the +// membership mirror from the persisted cursor until the stream is drained, +// over a plain postgres.js connection under bun — the SAME replay the +// Worker's every-minute cron runs (`src/auth/workos-events-replay.ts`). +// +// bun run db:drain-workos-events:prod # op run --env-file=.env.production +// +// Exists for the deploy gate (`scripts/ensure-workos-mirror-ready.ts`): the +// build that authorizes from the mirror trusts it only once the reconciler +// has drained the stream recently, and the gate must be able to MAKE that +// true itself rather than wait for a cron that may not be deployed yet — +// otherwise the reconciler build could only ever ship ahead of the gated +// one, by hand. Safe to run beside a live cron: a page is applied under the +// cursor's compare-and-set, so whichever run loses the stream writes +// nothing and stops. Runs until the stream is drained or another run owns +// it; a page budget bounds one pass, so a long backlog takes several. Exits +// 0 on a drain, 1 otherwise, with the reason. +// --------------------------------------------------------------------------- + +import { drizzle } from "drizzle-orm/postgres-js"; +import { Effect, Option } from "effect"; +import postgres from "postgres"; +import { WorkOS } from "@workos-inc/node"; + +import { makeUserStore } from "../src/auth/user-store"; +import { replayWorkOsEvents, type WorkOsEventsSyncReport } from "../src/auth/workos-events-replay"; +import { makeWorkOsMirrorStore } from "../src/auth/workos-mirror-store"; + +const connectionString = process.env.DATABASE_URL; +if (!connectionString) { + console.error("DATABASE_URL is not set"); + process.exit(1); +} +const apiKey = process.env.WORKOS_API_KEY; +if (!apiKey) { + console.error("WORKOS_API_KEY is not set"); + process.exit(1); +} + +const usesLocalDatabase = + connectionString.includes("127.0.0.1") || connectionString.includes("localhost"); + +const sql = postgres(connectionString, { + max: 1, + prepare: false, + ...(usesLocalDatabase ? {} : { ssl: "require" as const }), +}); +const db = drizzle(sql); +const workos = new WorkOS(apiKey); +const users = makeUserStore(db); + +// The script boundary: raw SDK / driver promises lifted once, here. Only a +// 404 is the deterministic "gone" the replay acts on; every other failure +// fails the pass, as in the Worker (`src/auth/workos-events-sync.ts`). +const fromPromise = (fn: () => Promise) => + Effect.tryPromise({ try: fn, catch: (cause) => cause }); + +const isNotFound = (cause: unknown): boolean => + typeof cause === "object" && + cause !== null && + "status" in cause && + (cause as { readonly status: unknown }).status === 404; + +const noneWhenGone = (fn: () => Promise) => + Effect.tryPromise({ try: fn, catch: (cause) => cause }).pipe( + Effect.map(Option.some), + Effect.catch((cause) => + isNotFound(cause) ? Effect.succeed(Option.none()) : Effect.fail(cause), + ), + ); + +// One pass is bounded by the replay's page budget; loop until the stream +// is drained, another run owns it, or the backfill has not run. +const MAX_PASSES = 50; + +const drain = Effect.gen(function* () { + const deps = { + source: { + listEvents: (options: Parameters[0]) => + fromPromise(async () => { + const page = await workos.events.listEvents({ + ...options, + events: [...options.events], + }); + return { data: page.data, after: page.listMetadata.after ?? null }; + }), + getOrganization: (organizationId: string) => + noneWhenGone(() => workos.organizations.getOrganization(organizationId)), + getUser: (userId: string) => noneWhenGone(() => workos.userManagement.getUser(userId)), + }, + store: { + getOrganization: (organizationId: string) => + fromPromise(() => users.getOrganization(organizationId)), + upsertOrganization: (organization: Parameters[0]) => + fromPromise(() => users.upsertOrganization(organization)), + getAccount: (accountId: string) => fromPromise(() => users.getAccount(accountId)), + }, + mirror: makeWorkOsMirrorStore(db), + }; + let last: WorkOsEventsSyncReport | null = null; + for (let pass = 0; pass < MAX_PASSES; pass++) { + const report = yield* replayWorkOsEvents(deps); + console.log( + `[drain-events] pass ${pass + 1}: ${report.pages} page(s), ${report.events} event(s), ` + + `${report.applied} applied, ${report.stale} stale, ${report.absent} absent — ${report.stopped}`, + ); + last = report; + if (report.stopped !== "page_budget") break; + } + return last; +}); + +const report = await Effect.runPromise( + drain.pipe(Effect.ensuring(Effect.promise(() => sql.end({ timeout: 5 })))), +); + +if (report === null || report.stopped !== "drained") { + console.error( + `[drain-events] the events stream was not drained: ${report?.stopped ?? "no pass ran"}` + + (report?.stopped === "awaiting_backfill" + ? " (run scripts/backfill-workos-mirror.ts first)" + : report?.stopped === "cursor_contended" + ? " (another run owns the stream; rerun once it finishes)" + : ""), + ); + process.exit(1); +} diff --git a/apps/cloud/src/account/org-api-key-revoke.node.test.ts b/apps/cloud/src/account/org-api-key-revoke.node.test.ts index 6682280a0..10353ccec 100644 --- a/apps/cloud/src/account/org-api-key-revoke.node.test.ts +++ b/apps/cloud/src/account/org-api-key-revoke.node.test.ts @@ -136,12 +136,14 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ deleteMembership: () => Effect.die("revoke does not write the membership mirror"), deleteUser: () => Effect.die("revoke does not write the membership mirror"), getCursor: () => Effect.die("revoke does not read the events cursor"), - setCursor: () => Effect.die("revoke does not move the events cursor"), + applyPage: () => Effect.die("revoke does not move the events cursor"), applyOrganizationScan: () => Effect.die("revoke does not run the backfill"), replayBoundary: () => Effect.die("revoke does not run the reconciler"), setReplayBoundary: () => Effect.die("revoke does not run the backfill"), backfillCompletedAt: () => Effect.die("revoke does not check mirror readiness"), markBackfillCompleted: () => Effect.die("revoke does not run the backfill"), + drainedAt: () => Effect.die("revoke does not check mirror readiness"), + markDrained: () => Effect.die("revoke does not run the reconciler"), organizationBackfilledAt: () => Effect.die("revoke does not report seats"), }); diff --git a/apps/cloud/src/auth/mirror-feeders.node.test.ts b/apps/cloud/src/auth/mirror-feeders.node.test.ts index 338ce4139..338194355 100644 --- a/apps/cloud/src/auth/mirror-feeders.node.test.ts +++ b/apps/cloud/src/auth/mirror-feeders.node.test.ts @@ -21,11 +21,12 @@ // writes nothing on a dry run, converges on a re-run, tombstones a // membership WorkOS no longer lists — but never one written after its // listing was taken — marks each org backfilled as of its listing, and -// records the events replay boundary only when it completes and only +// records the events replay boundary BEFORE its first listing and only // ONCE: a run that fails part-way keeps the marks of the orgs it -// finished and records no boundary, and a later completed run keeps the -// first boundary (the org renames and user deletions between two runs -// are the events stream's to replay) +// finished and the boundary it recorded, and its retry (like any later +// run) keeps that first boundary — so a user deleted between the failed +// attempt and the retry is still inside the events replay, and the +// reconciler clears their profile // - two scans of one org that overlap cannot resurrect a membership: a scan // that listed it, stalled, and resumed after a later listing (which no // longer had it) was applied is refused whole @@ -736,7 +737,7 @@ describe("backfill", () => { const backfilledAt = (org: string) => withMirror((mirror) => mirror.organizationBackfilledAt(org)); - it("records the replay boundary on first completion, marks and mirrors every organization's members, counts the writes, converges and repairs on a re-run", async () => { + it("records the replay boundary at its start, marks and mirrors every organization's members, counts the writes, converges and repairs on a re-run", async () => { const orgA = freshId("org"); const orgB = freshId("org"); await seedOrganization(orgA); @@ -785,6 +786,10 @@ describe("backfill", () => { expect(firstCompletion, "and that every organization is now covered").not.toBeNull(); expect(firstCompletion!.getTime()).toBeGreaterThanOrEqual(after!.getTime()); expect(after!.getTime()).toBeGreaterThanOrEqual(startedAt); + expect( + after!.getTime(), + "the boundary is the instant the run began reading, before any listing", + ).toBeLessThanOrEqual(startedAt + 60 * 1000); for (const org of [orgA, orgB]) { const marked = await backfilledAt(org); expect(marked, "each scanned organization is marked as of its listing").not.toBeNull(); @@ -918,31 +923,26 @@ describe("backfill", () => { ); }); - it("keeps the marks of the organizations it finished but records no replay boundary when a run fails part-way", async () => { + it("keeps the boundary a run that fails part-way recorded, so a user deleted before the retry is still the reconciler's to clear", async () => { const orgA = freshId("org"); const orgB = freshId("org"); await seedOrganization(orgA); await seedOrganization(orgB); - const member = freshId("user"); - const orgs = new Map([ - [orgA, [workosMembership(member, orgA)]], - [orgB, [workosMembership(member, orgB)]], + const staying = freshId("user"); + const deletedMeanwhile = freshId("user"); + await clearEventsRow(); + + // Attempt A mirrors both users of orgA, then fails on orgB's listing. + const attemptA = new Map([ + [orgA, [workosMembership(staying, orgA), workosMembership(deletedMeanwhile, orgA)]], + [orgB, [workosMembership(staying, orgB)]], ]); - // A completed run first, so there IS a boundary to protect. - await runBackfill(orgs, false); - const completed = await syncState(); - expect(completed).not.toBeNull(); - const markedA = await backfilledAt(orgA); - - // A re-run whose second organization fails on a WorkOS read: the first - // org was written, but the run as a whole did not complete — and even a - // completed one would keep the first boundary. const failing = { - ...source(orgs, []), + ...source(attemptA, []), listOrgMembers: (organizationId: string) => organizationId === orgB ? Effect.fail(new WorkOSError({ status: 503 })) - : Effect.succeed(orgs.get(organizationId) ?? []), + : Effect.succeed(attemptA.get(organizationId) ?? []), }; const exit = await withMirror((mirror) => Effect.exit( @@ -953,12 +953,58 @@ describe("backfill", () => { ), ); expect(Exit.isFailure(exit), "the run fails rather than skipping the org").toBe(true); - expect(await syncState(), "the completed run's boundary stands").toEqual(completed); - expect(await completedAt(), "and so does its completion mark").not.toBeNull(); + const boundary = await syncState(); + expect(boundary, "the failed attempt already fixed the replay boundary").not.toBeNull(); + expect(await backfilledAt(orgA), "the org it finished is marked").not.toBeNull(); + expect(await backfilledAt(orgB), "the org it did not reach is not").toBeNull(); + expect( + await completedAt(), + "no completion mark: the failed attempt did not cover every organization", + ).toBeNull(); + expect( + (await readMembers(orgA)).find((m) => m.accountId === deletedMeanwhile)?.email, + "the user's profile is mirrored", + ).toBe(`${deletedMeanwhile}@placeholder.test`); + + // WorkOS deletes `deletedMeanwhile` between the attempts. Its + // `user.deleted` event is stamped AFTER the boundary attempt A recorded. + const deletedAt = new Date(boundary!.getTime() + 1); + + // Retry B lists WorkOS without the deleted user and succeeds. + const attemptB = new Map([ + [orgA, [workosMembership(staying, orgA)]], + [orgB, [workosMembership(staying, orgB)]], + ]); + const retried = await runBackfill(attemptB, false); + expect(retried).toMatchObject({ organizations: 2, membershipsTombstoned: 1 }); + expect( + await syncState(), + "the retry keeps the first attempt's boundary instead of taking a later one", + ).toEqual(boundary); + expect( + await backfilledAt(orgB), + "and finishes the org the first attempt did not", + ).not.toBeNull(); + expect( + await completedAt(), + "the retry is the first run to cover every organization, so it records the completion", + ).not.toBeNull(); + // The scan tombstoned the membership, but the account profile is not + // the scan's to clear: that is the `user.deleted` event's job, which is + // exactly why the boundary must not move past it. + const tombstoned = (await readMembers(orgA)).find((m) => m.accountId === deletedMeanwhile); + expect(tombstoned?.status).toBe("inactive"); + expect(tombstoned?.email, "the profile is still there for the event to clear").not.toBeNull(); + + // The reconciler's first run reads from the kept boundary, so the + // deletion (stamped after it) is inside the replay and clears the row. + expect(deletedAt.getTime()).toBeGreaterThan(boundary!.getTime()); + const cleared = await withMirror((mirror) => mirror.deleteUser(deletedMeanwhile, deletedAt)); + expect(cleared).toBe(true); expect( - (await backfilledAt(orgA))!.getTime(), - "the org the failed run did finish is marked as of its new listing", - ).toBeGreaterThanOrEqual(markedA!.getTime()); + (await readMembers(orgA)).find((m) => m.accountId === deletedMeanwhile)?.email, + "the deleted user's profile is gone from the directory", + ).toBeNull(); }); it("scans one organization on demand and marks only that one", async () => { diff --git a/apps/cloud/src/auth/user-store.ts b/apps/cloud/src/auth/user-store.ts index 931d4fcc8..1dfdb7d02 100644 --- a/apps/cloud/src/auth/user-store.ts +++ b/apps/cloud/src/auth/user-store.ts @@ -41,12 +41,33 @@ export interface OrganizationPayload { export const organizationAcceptsName = (updatedAt: Date) => or(isNull(organizations.workosUpdatedAt), lte(organizations.workosUpdatedAt, updatedAt)); -export const makeUserStore = (db: DrizzleDb) => { - const getOrganization = async (id: string) => { - const rows = await db.select().from(organizations).where(eq(organizations.id, id)); - return rows[0] ?? null; - }; +const readOrganization = async (db: DrizzleDb, id: string) => { + const rows = await db.select().from(organizations).where(eq(organizations.id, id)); + return rows[0] ?? null; +}; +/** + * Insert the organization row for `row.id` with a freshly minted URL slug, + * and return the row now held for that id: the one inserted, or the one a + * concurrent writer minted first. THE single mint point for slugs: every + * organization row is born with one, so there is no nullable window and no + * self-healing. With `deletedAt` set this mints a TOMBSTONE — the row an + * organization deleted in WorkOS before the mirror ever saw it leaves + * behind, so a feeder still holding a membership of it cannot mint it live + * (`upsertOrganization` returns a marked row untouched). + * + * `ON CONFLICT DO NOTHING` (no target) absorbs BOTH unique violations + * without throwing: an id collision (the org was mirrored concurrently), + * which resolves to the row now held, and a slug collision (the candidate + * was claimed by a different org), which retries with a fresh candidate. + * + * @throws when slug minting exhausts its retries — `isTaken` is broken; + * surfacing loudly beats a silently unslugged organization. + */ +export const insertOrganization = async ( + db: DrizzleDb, + row: Pick, +): Promise => { const slugTaken = async (slug: string) => { const rows = await db .select({ id: organizations.id }) @@ -54,36 +75,36 @@ export const makeUserStore = (db: DrizzleDb) => { .where(eq(organizations.slug, slug)); return rows.length > 0; }; - - // Insert a brand-new org row carrying a freshly-minted slug. `ON CONFLICT DO - // NOTHING` (no target) absorbs BOTH unique violations without throwing: an - // id collision (the org was mirrored concurrently) and a slug collision (the - // candidate was claimed by a different org). Returns the inserted row, or - // null when either conflict swallowed the insert — the caller decides whether - // to re-read (id race) or retry with a new candidate (slug race). - const tryInsertOrg = async (org: OrganizationPayload, slug: string) => { - const [row] = await db + for (let attempt = 0; attempt < 4; attempt++) { + const slug = await generateOrgSlug(row.name, slugTaken); + const [inserted] = await db .insert(organizations) - .values({ - id: org.id, - name: org.name, - slug, - workosUpdatedAt: org.updatedAt, - }) + .values({ ...row, slug }) .onConflictDoNothing() .returning(); - return row ?? null; - }; + if (inserted) return inserted; + // The insert was swallowed by a conflict. If the id now exists, a + // concurrent writer mirrored it — return that row. Otherwise the slug + // candidate collided; loop and mint a fresh one. + const held = await readOrganization(db, row.id); + if (held) return held; + } + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: slug minting exhausted retries; surfacing loudly beats a silently unslugged org + throw new Error(`unable to mint a slug for organization ${row.id}`); +}; - // Every new org row is born with a slug — there is no nullable window and no - // self-healing. Existing rows keep their slug (stable across renames, so org - // URLs survive) and only refresh their name — and only from a payload at - // least as new as the one that last named it (`organizationAcceptsName`): - // a sign-in whose membership list was fetched before a rename would +export const makeUserStore = (db: DrizzleDb) => { + const getOrganization = (id: string) => readOrganization(db, id); + + // Existing rows keep their slug (stable across renames, so org URLs + // survive) and only refresh their name — and only from a payload at least + // as new as the one that last named it (`organizationAcceptsName`): a + // sign-in whose membership list was fetched before a rename would // otherwise revert the rename after it landed. A row marked deleted is // returned as it is: the organization is gone, and nothing a feeder still // holds about it (a name, a membership fetched before the deletion) is - // written — never re-minted live, never renamed. + // written — never re-minted live, never renamed. A row the mirror does not + // hold is minted live (`insertOrganization`). const upsertOrganization = async (org: OrganizationPayload) => { const existing = await getOrganization(org.id); if (existing) { @@ -95,18 +116,12 @@ export const makeUserStore = (db: DrizzleDb) => { .returning(); return updated ?? existing; } - for (let attempt = 0; attempt < 4; attempt++) { - const slug = await generateOrgSlug(org.name, slugTaken); - const inserted = await tryInsertOrg(org, slug); - if (inserted) return inserted; - // The insert was swallowed by a conflict. If the id now exists, a - // concurrent request mirrored it — return that row. Otherwise the slug - // candidate collided; loop and mint a fresh one. - const fresh = await getOrganization(org.id); - if (fresh) return fresh; - } - // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: slug minting exhausted retries; surfacing loudly beats a silently unslugged org - throw new Error(`unable to mint a slug for organization ${org.id}`); + return insertOrganization(db, { + id: org.id, + name: org.name, + workosUpdatedAt: org.updatedAt, + deletedAt: null, + }); }; return { diff --git a/apps/cloud/src/auth/workos-callback-state.node.test.ts b/apps/cloud/src/auth/workos-callback-state.node.test.ts index 44152dda4..a62f840b6 100644 --- a/apps/cloud/src/auth/workos-callback-state.node.test.ts +++ b/apps/cloud/src/auth/workos-callback-state.node.test.ts @@ -116,12 +116,14 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ deleteMembership: () => Effect.die("the callback does not delete memberships"), deleteUser: () => Effect.die("the callback does not delete users"), getCursor: () => Effect.die("the callback does not read the events cursor"), - setCursor: () => Effect.die("the callback does not move the events cursor"), + applyPage: () => Effect.die("the callback does not move the events cursor"), applyOrganizationScan: () => Effect.die("the callback does not run the backfill"), replayBoundary: () => Effect.die("the callback does not run the reconciler"), setReplayBoundary: () => Effect.die("the callback does not run the backfill"), backfillCompletedAt: () => Effect.die("the callback does not check mirror readiness"), markBackfillCompleted: () => Effect.die("the callback does not run the backfill"), + drainedAt: () => Effect.die("the callback does not check mirror readiness"), + markDrained: () => Effect.die("the callback does not run the reconciler"), organizationBackfilledAt: () => Effect.die("the callback does not report seats"), }); diff --git a/apps/cloud/src/auth/workos-events-replay.ts b/apps/cloud/src/auth/workos-events-replay.ts new file mode 100644 index 000000000..52032d748 --- /dev/null +++ b/apps/cloud/src/auth/workos-events-replay.ts @@ -0,0 +1,556 @@ +// --------------------------------------------------------------------------- +// The membership mirror's RECONCILER, as a pure function over its ports: +// replays the WorkOS Events API into the mirror store so changes made +// outside Executor — a member removed in the WorkOS dashboard, a role edited +// there, a profile updated, an SSO just-in-time join — land in the mirror +// without anyone signing in. +// +// Kept free of `cloudflare:workers` (no `DbService`, no `env`, no +// `WorkOSClient`), like `workos-mirror-store.ts` and +// `workos-mirror-backfill.ts`, so the SAME replay runs in three places: the +// Worker's every-minute cron and the signed webhook poke +// (`workos-events-sync.ts` binds the ports to the request-scoped services), +// and the deploy gate (`scripts/ensure-workos-mirror-ready.ts`, through +// `scripts/drain-workos-events.ts`) that must bring the mirror up to date +// BEFORE the build that authorizes from it goes live — and cannot wait on a +// cron that may not be deployed yet. The ports are the WorkOS reads a replay +// makes (`WorkOsEventsSource`), the organization and account rows it +// consults (`WorkOsEventsStore`), and the mirror store it writes. +// +// The Events API is the ONLY source this applies. It is ordered and +// replayable from an event id, so the mirror persists the id of the last +// event it applied (`workos_sync.cursor`) and resumes from there; a webhook +// delivery only pokes a run (`workos-webhook.ts`), it is never applied +// itself, because a webhook is unordered and at-least-once. Two runs may +// overlap (the every-minute cron, a webhook poke, the deploy gate), so a +// page is applied and its cursor advanced in ONE transaction that +// compare-and-sets the cursor first (`WorkOsMirrorShape.applyPage`): the run +// that lost the stream writes nothing. The `updatedAt` guard on upserts is +// not enough on its own — a lagging run replaying `membership.updated` after +// the leading run applied that membership's `deleted` would re-insert the +// revoked row. There is no first-run history replay: the one-off backfill +// (`scripts/backfill-workos-mirror.ts`) covers history, and its first run +// records the instant it began reading WorkOS as the REPLAY BOUNDARY +// (`workos_sync.range_start`) before it lists anything. A run with no cursor +// reads the stream from that boundary — never from a wall-clock guess, which +// would silently drop every revocation older than the guess — and with no +// boundary either it does nothing but warn: the backfill has not run, and +// there is no honest place to start. The boundary never moves: a backfill +// retry or re-run refreshes memberships only, so the organization renames +// and user deletions after the first boundary are this stream's alone to +// apply. +// +// A deletion event tombstones its row as of the event's own `createdAt`, +// not the payload's `updatedAt` (which predates the delete): the tombstone +// must be newer than every payload a feeder could have fetched before the +// delete, so none of them can reinstate the row. +// +// A page is PLANNED before its transaction opens: every event becomes a +// mirror write, and that planning is where the only WorkOS reads happen — +// resolving an organization the mirror has never seen, and reading the +// profile of a member the mirror has never seen. A membership event carries +// no profile, and the `user.created` that would have carried it may predate +// the replay boundary: a user who existed before the mirror shipped and +// joins an organization the backfill has already scanned gets a bare +// account row from the membership write, and nothing in the stream would +// ever fill it — the member would be unsearchable by name or email until an +// unrelated profile update or sign-in. So a membership created or updated +// for an account the mirror holds no profile for (no row, or the bare row a +// membership write mints) is planned WITH the profile (`UpsertMember`), one +// `getUser` per such member, never per event. A deterministic answer to +// either read ("WorkOS no longer has this organization / user") does not +// fail the run — a failed run re-reads the same page from the same cursor +// next tick, so one such event would freeze the whole mirror, including +// revocations in every other org — but it is not dropped either: a gone +// organization MARKS the organization deleted (below), the same write its +// own `organization.deleted` further down the stream makes; a gone user is +// mirrored without a profile, and their own `user.deleted` follows. +// +// `organization.deleted` MARKS the organization deleted +// (`organizations.deleted_at`, the same mark cloud's own deletion flow sets +// and its purge keeps as a tombstone): the mirror is the membership read +// path, so an org deleted in the WorkOS dashboard must stop authorizing its +// members' sessions here, and this event is the only way that reaches the +// mirror. The mark is written HERE, in the first build that consumes the +// event, so no `organization.deleted` is ever drained from the stream +// without effect — an event consumed before the mark existed could never be +// replayed. For the same reason an organization the mirror has never seen +// gets a TOMBSTONE row minted: with no row, a login that fetched its +// memberships before the deletion and stalled would mint the organization +// live afterwards, and nothing left in the stream would ever revoke it. It +// never PURGES: deleting tenant data and secrets is cloud's own flow +// (`db/org-deletion.ts`), sequenced with billing and confirmed by an admin, +// and an event must not do it. An org already marked (by cloud's own flow, +// or a replay) is `absent` and nothing changes. `organization.updated` renames an +// organization the mirror already holds — never inserts one, so a rename +// replayed after cloud purged the org cannot resurrect it with a fresh slug +// — under the same name guard every feeder applies, so a rename event and +// a sign-in's name order each other by their stamps however they arrive. +// --------------------------------------------------------------------------- + +import { Clock, Effect, Match, Option } from "effect"; +import type { Event as WorkOSEvent } from "@workos-inc/node/worker"; + +import type { Account, Organization, OrganizationPayload } from "./user-store"; +import type { WorkOSListEventsOptions } from "./workos"; +import { + WorkOsMirrorWrite, + mirrorMembershipFromWorkOs, + mirrorUserFromWorkOs, + type WorkOsMembershipPayload, + type WorkOsMirrorShape, + type WorkOsMirrorUser, + type WorkOsMirrorWriteOutcome, + type WorkOsUserPayload, +} from "./workos-mirror-store"; + +/** + * The event types the mirror follows. Invitations are not mirrored (they + * stay a live WorkOS read), and `organization.created` is not needed: an org + * is mirrored lazily the first time a membership or a session names it. + */ +export const MIRRORED_EVENT_NAMES = [ + "user.created", + "user.updated", + "user.deleted", + "organization_membership.created", + "organization_membership.updated", + "organization_membership.deleted", + "organization.updated", + "organization.deleted", +] as const; + +export type WorkOsMirroredEventName = (typeof MIRRORED_EVENT_NAMES)[number]; + +/** The SDK events the reconciler applies, narrowed to the followed types. */ +export type WorkOsMirroredEvent = Extract; + +const mirroredEventNames: ReadonlySet = new Set(MIRRORED_EVENT_NAMES); + +/** Whether an event from the stream is one the mirror follows. */ +export const isMirroredEvent = (event: WorkOSEvent): event is WorkOsMirroredEvent => + mirroredEventNames.has(event.event); + +/** + * What one event did to the mirror: + * - `applied`: a row was written, marked, or tombstoned; + * - `stale`: the `updatedAt` guard refused an older payload (a replay or a + * late event behind a fresher write); + * - `absent`: a delete found its row already tombstoned or superseded by a + * newer membership (a replayed delete), a rename found no live + * organization row — the mirror has never seen it, or it is marked + * deleted — or a deletion mark found the organization already marked. + */ +export type WorkOsEventOutcome = WorkOsMirrorWriteOutcome; + +/** One page of the Events API stream, as the source hands it to the replay. */ +export interface WorkOsEventsPage { + readonly data: readonly WorkOSEvent[]; + /** The id to resume after, or `null` at the end of the stream. */ + readonly after: string | null; +} + +/** The WorkOS organization fields the replay reads to mint an org row. */ +export interface WorkOsOrganizationPayload { + readonly id: string; + readonly name: string; + readonly updatedAt: string; +} + +/** + * The WorkOS reads one replay makes, over whatever client the caller wires: + * the Events API page, and — only for a membership event whose organization + * or member the mirror has never seen — the organization or user resource. + * The two lookups answer `None` when WorkOS no longer has the resource (a + * 404): that is a deterministic answer the replay acts on, not a failure. + * Every other failure (401/403, 429, 5xx, no answer) is `E` and fails the + * run, so the event is retried once the cause is fixed rather than skipped. + */ +export interface WorkOsEventsSource { + readonly listEvents: (options: WorkOSListEventsOptions) => Effect.Effect; + readonly getOrganization: ( + organizationId: string, + ) => Effect.Effect, E>; + readonly getUser: (userId: string) => Effect.Effect, E>; +} + +/** + * The organization and account rows a replay consults while planning a + * page: the org row a membership's foreign key needs (minted from WorkOS + * through `upsertOrganization` when the mirror has never seen it — the one + * slug mint point) and the account row that says whether the member's + * profile is already held. + */ +export interface WorkOsEventsStore { + readonly getOrganization: (organizationId: string) => Effect.Effect; + readonly upsertOrganization: ( + organization: OrganizationPayload, + ) => Effect.Effect; + readonly getAccount: (accountId: string) => Effect.Effect; +} + +/** Everything one replay reads and writes. */ +export interface WorkOsEventsReplayDeps { + readonly source: WorkOsEventsSource; + readonly store: WorkOsEventsStore; + readonly mirror: WorkOsMirrorShape; +} + +/** A membership event's payload: the SDK's `OrganizationMembership`, which names its organization. */ +interface WorkOsMembershipEventPayload extends WorkOsMembershipPayload { + readonly organizationName: string; +} + +// The organization row a membership event needs: the mirror's, or — for an +// org the mirror has never seen (created and populated in the WorkOS +// dashboard before anyone signed in) — minted from the WorkOS organization +// so the membership's foreign key holds. `None` when WorkOS no longer has +// the organization. +const resolveOrganization = ( + deps: WorkOsEventsReplayDeps, + organizationId: string, +): Effect.Effect, E> => + Effect.gen(function* () { + const existing = yield* deps.store.getOrganization(organizationId); + if (existing) return Option.some(existing); + const fresh = yield* deps.source.getOrganization(organizationId); + if (Option.isNone(fresh)) return Option.none(); + const minted = yield* deps.store.upsertOrganization({ + id: fresh.value.id, + name: fresh.value.name, + updatedAt: new Date(fresh.value.updatedAt), + }); + return Option.some(minted); + }); + +// A membership event carries only the organization's id, so an org the +// mirror has never seen is mirrored first (`resolveOrganization`) so the +// membership's foreign key holds. That goes for a DELETE too: it leaves a +// tombstone behind even when the mirror has never seen the membership (so +// the backfill's older payload cannot insert it live), and the tombstone row +// needs the org as much as a live one. An org WorkOS no longer has (deleted +// there, or through Executor, after this event was emitted) is MARKED +// deleted instead — minting its tombstone row when the mirror has never +// seen it — so a login still holding a membership of it cannot mint it +// live: its own `organization.deleted` follows in the stream and finds the +// mark already there, and the membership itself is not written, there is +// nothing live to hold it. +const planMembershipWrite = ( + deps: WorkOsEventsReplayDeps, + membership: WorkOsMembershipEventPayload, + event: { readonly id: string; readonly createdAt: string }, + write: () => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const organization = yield* resolveOrganization(deps, membership.organizationId); + if (Option.isNone(organization)) { + yield* Effect.logWarning( + "workos_events: membership for an organization WorkOS no longer has; marking the org deleted instead", + { organizationId: membership.organizationId, eventId: event.id }, + ); + return WorkOsMirrorWrite.MarkOrganizationDeleted({ + organizationId: membership.organizationId, + name: membership.organizationName, + deletedAt: new Date(event.createdAt), + }); + } + return yield* write(); + }); + +// Whether the mirror holds a profile for the account: none at all, or only +// the bare row a membership write mints (`ensureAccount`: no email, no +// stamp), calls for a WorkOS read. A deletion tombstone (no email, stamped +// by `deleteUser`) does not: WorkOS never reuses a user id, and the +// membership write refuses the account anyway. +const holdsNoProfile = (account: Account | null): boolean => + account === null || (account.email === null && account.workosUpdatedAt === null); + +/** + * The user ids whose profile an earlier event of the SAME page has already + * planned a write for (`user.created` / `user.updated`, or a profile read + * for a membership). A page is planned in full before it is applied, so the + * mirror does not yet hold what the page's own earlier events carry; this + * is what keeps a `user.created` followed by that user's membership in one + * page from reading the profile WorkOS just streamed. + */ +export type PlannedProfiles = Set; + +// The member's profile from WorkOS, when neither the mirror nor an earlier +// event of the page holds one (see the header); `null` when one does, or +// when WorkOS no longer has the user (the user's own `user.deleted` follows +// in the stream, or has been applied). +const planMemberProfile = ( + deps: WorkOsEventsReplayDeps, + userId: string, + event: { readonly id: string }, + profiled: PlannedProfiles, +): Effect.Effect => + Effect.gen(function* () { + if (profiled.has(userId)) return null; + const account = yield* deps.store.getAccount(userId); + if (!holdsNoProfile(account)) return null; + const user = yield* deps.source.getUser(userId); + if (Option.isNone(user)) { + yield* Effect.logWarning( + "workos_events: membership for a user WorkOS no longer has; mirrored without a profile", + { eventId: event.id }, + ); + return null; + } + profiled.add(userId); + return mirrorUserFromWorkOs(user.value); + }); + +const planMembershipUpsert = ( + deps: WorkOsEventsReplayDeps, + membership: WorkOsMembershipEventPayload, + event: { readonly id: string; readonly createdAt: string }, + profiled: PlannedProfiles, +) => + planMembershipWrite(deps, membership, event, () => + Effect.map(planMemberProfile(deps, membership.userId, event, profiled), (user) => + user === null + ? WorkOsMirrorWrite.UpsertMembership({ + membership: mirrorMembershipFromWorkOs(membership), + }) + : WorkOsMirrorWrite.UpsertMember({ + user, + membership: mirrorMembershipFromWorkOs(membership), + }), + ), + ); + +/** + * Translate one event into the mirror write it calls for. Every followed + * event yields a write: none is drained from the stream without effect. + * This is the only step that may read WorkOS (an organization the mirror + * has never seen, a member it holds no profile for); it runs before the + * page's transaction opens. Fails on a store failure or a WorkOS failure + * that a retry could clear — the run stops before the page is applied, so + * the event is retried next run. `profiled` is the page's running set of + * users whose profile is already planned (see {@link PlannedProfiles}); one + * set per page. + */ +export const planWorkOsEvent = ( + deps: WorkOsEventsReplayDeps, + event: WorkOsMirroredEvent, + profiled: PlannedProfiles = new Set(), +): Effect.Effect => { + const userWrite = (data: WorkOsUserPayload) => + Effect.sync(() => { + profiled.add(data.id); + return WorkOsMirrorWrite.UpsertUser({ user: mirrorUserFromWorkOs(data) }); + }); + return Match.value(event).pipe( + Match.discriminatorsExhaustive("event")({ + "user.created": ({ data }) => userWrite(data), + "user.updated": ({ data }) => userWrite(data), + "user.deleted": ({ data }) => + Effect.succeed( + WorkOsMirrorWrite.DeleteUser({ + accountId: data.id, + deletedAt: new Date(event.createdAt), + }), + ), + "organization_membership.created": ({ data }) => + planMembershipUpsert(deps, data, event, profiled), + "organization_membership.updated": ({ data }) => + planMembershipUpsert(deps, data, event, profiled), + "organization_membership.deleted": ({ data }) => + planMembershipWrite(deps, data, event, () => + Effect.succeed( + WorkOsMirrorWrite.DeleteMembership({ + membership: { + id: data.id, + accountId: data.userId, + organizationId: data.organizationId, + }, + deletedAt: new Date(event.createdAt), + }), + ), + ), + "organization.updated": ({ data }) => + Effect.succeed( + WorkOsMirrorWrite.RenameOrganization({ + organizationId: data.id, + name: data.name, + updatedAt: new Date(data.updatedAt), + }), + ), + "organization.deleted": ({ data }) => + Effect.logWarning( + "workos_events: organization.deleted received; marking the org deleted locally — tenant data is kept (purging is cloud's own flow, db/org-deletion.ts)", + { organizationId: data.id, eventId: event.id }, + ).pipe( + Effect.as( + WorkOsMirrorWrite.MarkOrganizationDeleted({ + organizationId: data.id, + name: data.name, + deletedAt: new Date(event.createdAt), + }), + ), + ), + }), + ); +}; + +// One page is one WorkOS read and one cursor advance. 100 is the API's +// maximum; the page budget bounds a single run (a backlog after an outage +// drains over successive runs, each committing what it applied) so a cron +// invocation stays well inside the Worker's wall-clock limits. +const PAGE_SIZE = 100; +const MAX_PAGES_PER_RUN = 20; + +export interface WorkOsEventsSyncReport { + readonly pages: number; + readonly events: number; + readonly applied: number; + readonly stale: number; + readonly absent: number; + /** + * Why the run ended: the stream was read to its end (`drained`), another + * run moved the cursor first (`cursor_contended`), the page budget for one + * run was spent with more to read (`page_budget`), or there is neither a + * cursor nor a replay boundary to start from — the backfill has not run — + * so nothing was read (`awaiting_backfill`). + */ + readonly stopped: "drained" | "cursor_contended" | "page_budget" | "awaiting_backfill"; + /** The cursor this run left behind (the last event id it committed). */ + readonly cursor: string | null; +} + +/** + * One reconciler run: read the cursor (or, before the first page was ever + * committed, the backfill's replay boundary), page the Events API from it + * (oldest first), plan every event, and apply each page with its cursor + * advance in one transaction. Stops as soon as that transaction finds the + * cursor moved — another run owns the stream, and nothing from the page was + * written — and fails (before the page is applied) on the first source, + * store, or mirror failure, so nothing is skipped: the next run resumes + * from the last committed page. With neither cursor nor boundary it reads + * nothing and reports `awaiting_backfill`. A run that reads the stream to + * its end records the drain (`markDrained`) as of its own start. + */ +export const replayWorkOsEvents = (deps: WorkOsEventsReplayDeps) => + Effect.gen(function* () { + const { source, mirror } = deps; + + // Taken before the first read, so the drained mark below cannot + // post-date an event this run never saw. + const startedAt = new Date(yield* Clock.currentTimeMillis); + let cursor = yield* mirror.getCursor(); + const counts = { + pages: 0, + events: 0, + applied: 0, + stale: 0, + absent: 0, + }; + let stopped: WorkOsEventsSyncReport["stopped"] = "page_budget"; + + // Where the next page starts: after the last committed event id, or — + // for the very first read, which has no id to resume from — at the + // backfill's replay boundary, the only instant known to be covered. + let resume: { readonly after: string } | { readonly rangeStart: string }; + if (cursor === null) { + const boundary = yield* mirror.replayBoundary(); + if (boundary === null) { + yield* Effect.logWarning( + "workos_events: no cursor and no replay boundary — the mirror backfill has not run (db:backfill-workos-mirror:prod); nothing read", + ); + const report: WorkOsEventsSyncReport = { + ...counts, + stopped: "awaiting_backfill", + cursor, + }; + return report; + } + resume = { rangeStart: boundary.toISOString() }; + } else { + resume = { after: cursor }; + } + + while (counts.pages < MAX_PAGES_PER_RUN) { + const page = yield* source.listEvents({ + events: MIRRORED_EVENT_NAMES, + limit: PAGE_SIZE, + order: "asc", + ...resume, + }); + counts.pages += 1; + if (page.data.length === 0) { + stopped = "drained"; + break; + } + + // Plan first (the WorkOS reads), then apply under the cursor lock. + let lastEventId = cursor; + const profiled: PlannedProfiles = new Set(); + const planned: { + readonly event: WorkOsMirroredEvent; + readonly write: WorkOsMirrorWrite; + }[] = []; + for (const event of page.data) { + counts.events += 1; + lastEventId = event.id; + if (!isMirroredEvent(event)) { + // The request named the followed types; anything else is a WorkOS + // change of contract worth seeing, not a reason to stop the stream. + yield* Effect.logWarning("workos_events: unrequested event type skipped", { + event: event.event, + eventId: event.id, + }); + continue; + } + planned.push({ event, write: yield* planWorkOsEvent(deps, event, profiled) }); + } + + // `lastEventId` is an event id here: the page was non-empty. + if (lastEventId === null) break; + const outcomes = yield* mirror.applyPage( + cursor, + lastEventId, + planned.map((p) => p.write), + ); + if (Option.isNone(outcomes)) { + yield* Effect.logWarning("workos_events: cursor moved by another run; stopping", { + expected: cursor, + }); + stopped = "cursor_contended"; + break; + } + for (const [index, outcome] of outcomes.value.entries()) { + counts[outcome] += 1; + if (outcome === "absent") { + // Normal for a replayed delete; for a rename it means the org was + // never mirrored or is marked deleted, and for a deletion mark + // that it is already marked — either way nothing to do. + yield* Effect.logInfo("workos_events: event targets a row the mirror does not hold", { + event: planned[index]?.event.event, + eventId: planned[index]?.event.id, + }); + } + } + cursor = lastEventId; + resume = { after: cursor }; + if (page.after === null) { + stopped = "drained"; + break; + } + } + + if (stopped === "drained") { + // The stream was read to its end: everything WorkOS had emitted by + // the time this run began is now in the mirror. Recorded as of the + // run's START, not its end — an event emitted while the run was + // reading may still be ahead of the last page it saw — so the mark + // never claims more than was covered. This is what the authorization + // path reads to tell a caught-up mirror from one whose reconciler has + // stalled. + yield* mirror.markDrained(startedAt); + } + + const report: WorkOsEventsSyncReport = { ...counts, stopped, cursor }; + yield* Effect.logInfo("workos_events: sync run finished", report); + return report; + }).pipe(Effect.withSpan("workos_events.replay")); diff --git a/apps/cloud/src/auth/workos-events-runner.ts b/apps/cloud/src/auth/workos-events-runner.ts new file mode 100644 index 000000000..1e1c860e7 --- /dev/null +++ b/apps/cloud/src/auth/workos-events-runner.ts @@ -0,0 +1,56 @@ +// --------------------------------------------------------------------------- +// Runs one reconciler pass (`syncWorkOsEvents`) from a Worker entry that is +// not an HTTP request handled by the Effect app: the every-minute cron +// (`scheduled` in server.ts) and the webhook poke (`workos-webhook.ts`, +// detached past the response with `waitUntil`). +// +// Both entries build the request-scoped services FRESH for the run — the +// same reason `mcp/auth.ts` does: a postgres socket belongs to one Workers +// invocation, and the webhook route's own per-request layer is closed the +// moment its response is returned, so a detached run cannot borrow it. The +// run is its own scope; the socket is released when it ends. +// +// A failing run is captured (Sentry + structured log) and swallowed here: +// neither entry has a caller to report to, and the run is retried by the +// next cron tick from the last committed cursor. +// --------------------------------------------------------------------------- + +import { Effect, Layer } from "effect"; + +import { captureCauseEffect } from "../observability"; +import { WorkerTelemetryLive } from "../observability/telemetry"; +import { makeDbLayer } from "../db/db"; +import { makeUserStoreLayer } from "./context"; +import { CoreSharedServices } from "./workos"; +import { syncWorkOsEvents } from "./workos-events-sync"; +import { makeWorkOsMirrorLayer } from "./workos-mirror"; + +const makeSyncServices = () => { + const dbLive = makeDbLayer(); + return Layer.mergeAll( + makeUserStoreLayer().pipe(Layer.provide(dbLive)), + makeWorkOsMirrorLayer().pipe(Layer.provide(dbLive)), + CoreSharedServices, + ); +}; + +/** + * One reconciler pass over fresh request-scoped services. Resolves when the + * pass ends, whether it drained the stream, stopped at the page budget, + * yielded to another run, or failed (a failure is reported, never thrown). + */ +export const runWorkOsEventsSync = (): Promise => + Effect.runPromise( + syncWorkOsEvents().pipe( + Effect.asVoid, + Effect.provide(makeSyncServices()), + Effect.scoped, + Effect.catchCause((cause) => + Effect.gen(function* () { + yield* Effect.logError("workos_events: sync run failed", cause); + yield* captureCauseEffect(cause); + }), + ), + Effect.provide(WorkerTelemetryLive), + ), + ); diff --git a/apps/cloud/src/auth/workos-events-sync.node.test.ts b/apps/cloud/src/auth/workos-events-sync.node.test.ts new file mode 100644 index 000000000..d0a465a09 --- /dev/null +++ b/apps/cloud/src/auth/workos-events-sync.node.test.ts @@ -0,0 +1,1268 @@ +// --------------------------------------------------------------------------- +// The membership mirror's RECONCILER (`workos-events-sync.ts`) and the +// webhook that pokes it (`workos-webhook.ts`), against the real PGlite +// Postgres every cloud unit test runs on (scripts/test-globalsetup.ts). +// WorkOS is a fake `WorkOSClient` for the Events API (the emulator has no +// events route); the signature check runs the REAL client's verifier over a +// locally computed HMAC, because that check is the webhook's only +// authentication. +// +// What this pins: +// - every followed event type lands in the mirror: user created/updated/ +// deleted, membership created/updated/deleted, organization renamed +// - an older event never regresses a newer row (`stale`) — an older +// organization rename included — a replayed delete is `absent`, and +// `organization.deleted` MARKS the org deleted without purging anything; +// replayed, or after cloud's own flow marked it first, it is `absent` +// - `organization.deleted` for an org the mirror has never seen MINTS a +// tombstone row, so a login that fetched a membership of it before the +// deletion cannot mint the org or the membership afterwards +// - a delete TOMBSTONES its row as of the event's `createdAt`, so an older +// payload replayed after it is `stale` and the row stays inactive — even +// when the delete arrives before the mirror has ever seen the membership +// (the reconciler ahead of the backfill): the tombstone is minted, with +// its organization, so the backfill cannot insert the row live +// - a membership created or updated for a member the mirror holds no +// profile for reads the profile from WorkOS (one `getUser`, only then), +// so a pre-boundary user who joins a scanned org is searchable by name +// and email; a member WorkOS no longer has is mirrored bare, and a +// transient failure fails the run +// - a membership for an organization the mirror has never seen mirrors +// the org first (one WorkOS read), so the foreign key holds; one whose +// org WorkOS no longer has marks the org deleted instead (minting the +// tombstone) and the cursor still advances, while a transient WorkOS +// failure still fails the run +// - `organization.updated` never inserts an org the mirror does not hold +// - a run with no cursor reads from the backfill's replay boundary, and +// with no boundary either reads nothing (the backfill has not run) +// - a run pages from the persisted cursor, commits after every page, and +// STOPS when another run moves the cursor under it — with NOTHING from +// the contended page written (a lagging run cannot resurrect a +// membership the leading run already deleted) +// - a run that reads the stream to its end records the drain as of its +// start (the authorization path's caught-up check); a run that read +// nothing, or yielded the stream, records none +// - the webhook accepts only a genuinely signed delivery, never applies +// it, and refuses everything when no signing secret is configured +// --------------------------------------------------------------------------- + +import { createHmac } from "node:crypto"; + +import { describe, expect, it } from "@effect/vitest"; +import { sql } from "drizzle-orm"; +import { Effect, Exit, Layer, Option } from "effect"; +import { HttpRouter, HttpServer } from "effect/unstable/http"; +import type { Organization, OrganizationMembership, User } from "@workos-inc/node/worker"; + +import { MemberDirectory, type MemberStatus } from "@executor-js/api/server"; + +import { DbService } from "../db/db"; +import { UserStoreService } from "./context"; +import { WorkOSError } from "./errors"; +import { cloudMemberDirectoryLayer } from "./member-directory"; +import { mirrorSignIn } from "./mirror-feeders"; +import { WorkOSClient, type WorkOSClientService, type WorkOSListEventsOptions } from "./workos"; +import { + planEvent, + syncWorkOsEvents, + type WorkOsEventOutcome, + type WorkOsEventsSyncReport, + type WorkOsMirroredEvent, +} from "./workos-events-sync"; +import { WorkOsMirror, WorkOsMirrorWrite, mirrorMembershipFromWorkOs } from "./workos-mirror"; +import { WORKOS_WEBHOOK_PATH, makeWorkOsWebhookRoute } from "./workos-webhook"; + +const T1 = "2026-01-01T00:00:00.000Z"; +const T2 = "2026-01-02T00:00:00.000Z"; +const T3 = "2026-01-03T00:00:00.000Z"; + +// Synthetic identities only; every test mints its own ids so the shared test +// database never couples two tests. +const freshId = (prefix: string) => `${prefix}_${crypto.randomUUID().replaceAll("-", "")}`; + +const workosUser = (id: string, overrides: Partial = {}): User => ({ + object: "user", + id, + email: `${id}@placeholder.test`, + emailVerified: true, + firstName: "Ada", + lastName: "Placeholder", + profilePictureUrl: null, + lastSignInAt: T1, + locale: null, + createdAt: T1, + updatedAt: T1, + externalId: null, + metadata: {}, + ...overrides, +}); + +const workosMembership = ( + userId: string, + organizationId: string, + overrides: Partial = {}, +): OrganizationMembership => ({ + object: "organization_membership", + id: `om_${userId}_${organizationId}`, + userId, + organizationId, + organizationName: `Org ${organizationId}`, + status: "active", + directoryManaged: false, + createdAt: T1, + updatedAt: T1, + customAttributes: {}, + role: { slug: "member" }, + ...overrides, +}); + +const workosOrganization = (id: string, name: string, updatedAt = T1): Organization => ({ + object: "organization", + id, + name, + allowProfilesOutsideOrganization: false, + domains: [], + createdAt: T1, + updatedAt, + externalId: null, + metadata: {}, +}); + +const userEvent = ( + event: "user.created" | "user.updated" | "user.deleted", + data: User, + id = freshId("event"), + createdAt = T1, +): WorkOsMirroredEvent => ({ + id, + event, + data, + createdAt, + context: undefined, +}); + +const membershipEvent = ( + event: + | "organization_membership.created" + | "organization_membership.updated" + | "organization_membership.deleted", + data: OrganizationMembership, + id = freshId("event"), + createdAt = T1, +): WorkOsMirroredEvent => ({ + id, + event, + data, + createdAt, + context: undefined, +}); + +const organizationEvent = ( + event: "organization.updated" | "organization.deleted", + data: Organization, + id = freshId("event"), + createdAt = T1, +): WorkOsMirroredEvent => ({ + id, + event, + data, + createdAt, + context: undefined, +}); + +/** + * A `WorkOSClient` whose every method is one of `methods`; anything else is + * an unexpected call and dies, so a reconciler that silently adds a WorkOS + * read fails the test instead of passing on a fake. + */ +const stubWorkOS = (methods: Partial) => + Layer.succeed( + WorkOSClient, + new Proxy({} as WorkOSClientService, { + get: (_target, prop) => + (methods as Record)[prop] ?? + (() => Effect.die(`unexpected WorkOSClient.${String(prop)} call`)), + }), + ); + +/** + * `getUser` for every member a test's membership events name: the + * reconciler reads a profile for a member the mirror holds none for, and + * the strict stub above would die on it. Records each read in `reads`. + */ +const profiles = (reads: string[] = []): Partial => ({ + getUser: (userId) => + Effect.sync(() => { + reads.push(userId); + return workosUser(userId); + }), +}); + +const DbLive = DbService.Live; +const MirrorServices = Layer.mergeAll( + WorkOsMirror.Live, + UserStoreService.Live, + cloudMemberDirectoryLayer, +).pipe(Layer.provideMerge(DbLive)); + +type Services = WorkOsMirror | UserStoreService | MemberDirectory | DbService | WorkOSClient; + +const run = ( + body: Effect.Effect, + workos: Layer.Layer = stubWorkOS({}), +) => + Effect.runPromise( + body.pipe(Effect.provide(Layer.mergeAll(MirrorServices, workos)), Effect.scoped), + ); + +const seedOrganization = (id: string) => + Effect.flatMap(UserStoreService.asEffect(), (users) => + users.use("upsertOrganization", (s) => + s.upsertOrganization({ id, name: `Org ${id}`, updatedAt: new Date(T1) }), + ), + ); + +const readOrganization = (id: string) => + Effect.flatMap(UserStoreService.asEffect(), (users) => + users.use("getOrganization", (s) => s.getOrganization(id)), + ); + +const readMembership = ( + accountId: string, + organizationId: string, + statuses?: readonly MemberStatus[], +) => + Effect.flatMap(MemberDirectory.asEffect(), (directory) => + directory.membership(accountId, organizationId, statuses), + ); + +/** + * Apply one event the way a run does — plan it, then apply it as a one-event + * page under the cursor CAS — and report the event's outcome. The cursor is + * instance-wide; each apply moves it to a fresh id, which is what a run does. + */ +const applyEvent = (event: WorkOsMirroredEvent) => + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const write = yield* planEvent(event); + const prev = yield* mirror.getCursor(); + const outcomes = yield* mirror.applyPage(prev, freshId("event"), [write]); + expect(Option.isSome(outcomes), "no other run contends in a single-event apply").toBe(true); + const outcome: WorkOsEventOutcome | undefined = Option.getOrElse(outcomes, () => [])[0]; + expect(outcome, "one write, one outcome").toBeDefined(); + return outcome ?? "absent"; + }); + +describe("applyEvent", () => { + it("mirrors a user, refreshes it, refuses an older update, and deletes it", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(userId, org)), + ); + + const created = yield* applyEvent( + userEvent("user.created", workosUser(userId, { firstName: "Grace", updatedAt: T2 })), + ); + const afterCreate = yield* readMembership(userId, org); + const updated = yield* applyEvent( + userEvent("user.updated", workosUser(userId, { firstName: "Newer", updatedAt: T3 })), + ); + const afterUpdate = yield* readMembership(userId, org); + const stale = yield* applyEvent( + userEvent("user.updated", workosUser(userId, { firstName: "Stale", updatedAt: T1 })), + ); + const afterStale = yield* readMembership(userId, org); + // The delete is stamped with the EVENT's time (T3), after every + // payload above; the SDK payload's own `updatedAt` predates it. + const deleted = yield* applyEvent( + userEvent("user.deleted", workosUser(userId), freshId("event"), T3), + ); + const afterDelete = yield* readMembership(userId, org); + const tombstoned = yield* readMembership(userId, org, ["inactive"]); + // A profile update that happened before the delete but lands after it. + const lateUpdate = yield* applyEvent( + userEvent("user.updated", workosUser(userId, { firstName: "Late", updatedAt: T2 })), + ); + const afterLate = yield* readMembership(userId, org, ["inactive"]); + return { + created, + afterCreate, + updated, + afterUpdate, + stale, + afterStale, + deleted, + afterDelete, + tombstoned, + lateUpdate, + afterLate, + }; + }), + stubWorkOS(profiles()), + ); + expect(result.created).toBe("applied"); + expect(result.afterCreate?.name).toBe("Grace Placeholder"); + expect(result.updated).toBe("applied"); + expect(result.afterUpdate?.name).toBe("Newer Placeholder"); + expect(result.stale, "an event older than the stored row is reported stale").toBe("stale"); + expect(result.afterStale?.name, "and leaves the newer row untouched").toBe("Newer Placeholder"); + expect(result.deleted).toBe("applied"); + expect(result.afterDelete, "deleting the user tombstones its membership").toBeNull(); + expect(result.tombstoned, "the row stays, inactive, with the profile cleared").toMatchObject({ + status: "inactive", + name: null, + email: null, + }); + expect(result.lateUpdate, "a payload older than the deletion is refused").toBe("stale"); + expect(result.afterLate).toMatchObject({ status: "inactive", name: null }); + }); + + it("mirrors a membership, updates its role, refuses an older update, and deletes it", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + const created = yield* applyEvent( + membershipEvent( + "organization_membership.created", + workosMembership(userId, org, { status: "pending" }), + ), + ); + const afterCreate = yield* readMembership(userId, org); + const updated = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { + role: { slug: "admin" }, + updatedAt: T3, + }), + ), + ); + const afterUpdate = yield* readMembership(userId, org); + const stale = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { + role: { slug: "member" }, + status: "inactive", + updatedAt: T2, + }), + ), + ); + const afterStale = yield* readMembership(userId, org); + // The delete is stamped with the EVENT's time (T3), after every + // payload above; the SDK payload's own `updatedAt` predates it. + const deleted = yield* applyEvent( + membershipEvent( + "organization_membership.deleted", + workosMembership(userId, org), + freshId("event"), + T3, + ), + ); + const afterDelete = yield* readMembership(userId, org); + const deletedAgain = yield* applyEvent( + membershipEvent( + "organization_membership.deleted", + workosMembership(userId, org), + freshId("event"), + T3, + ), + ); + // A membership update that happened before the delete but lands + // after it (a lagging feeder, an out-of-order delivery): refused, the + // tombstone stands. + const lateUpdate = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { + role: { slug: "admin" }, + updatedAt: T2, + }), + ), + ); + const afterLate = yield* readMembership(userId, org, ["inactive"]); + return { + created, + afterCreate, + updated, + afterUpdate, + stale, + afterStale, + deleted, + afterDelete, + deletedAgain, + lateUpdate, + afterLate, + }; + }), + stubWorkOS(profiles()), + ); + expect(result.created).toBe("applied"); + expect(result.afterCreate).toMatchObject({ + membershipId: `om_${userId}_${org}`, + status: "pending", + role: "member", + }); + expect(result.updated).toBe("applied"); + expect(result.afterUpdate).toMatchObject({ + status: "active", + role: "admin", + }); + expect(result.stale).toBe("stale"); + expect(result.afterStale).toMatchObject({ + status: "active", + role: "admin", + }); + expect(result.deleted).toBe("applied"); + expect(result.afterDelete, "a tombstone reads as no membership").toBeNull(); + expect(result.deletedAgain, "a replayed delete changes nothing").toBe("absent"); + expect(result.lateUpdate, "a payload older than the deletion is refused").toBe("stale"); + expect(result.afterLate).toMatchObject({ + status: "inactive", + role: "admin", + }); + }); + + it("tombstones a membership the mirror has never seen, mirroring its organization first, so the backfill cannot insert it live", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const reads: string[] = []; + const result = await run( + Effect.gen(function* () { + // The deletion lands before any feeder wrote the membership or the + // org; the org is read from WorkOS so the tombstone row can exist. + const deleted = yield* applyEvent( + membershipEvent( + "organization_membership.deleted", + workosMembership(userId, org), + freshId("event"), + T2, + ), + ); + const organization = yield* readOrganization(org); + const tombstone = yield* readMembership(userId, org, ["inactive"]); + // The backfill, listing WorkOS as it was before the deletion, writes + // the membership afterwards: refused, the tombstone stands. + const mirror = yield* WorkOsMirror; + const backfilled = yield* mirror.upsertMembership( + mirrorMembershipFromWorkOs(workosMembership(userId, org)), + ); + const afterBackfill = yield* readMembership(userId, org); + return { deleted, organization, tombstone, backfilled, afterBackfill }; + }), + stubWorkOS({ + getOrganization: (id) => { + reads.push(id); + return Effect.succeed(workosOrganization(id, "Dashboard Org")); + }, + }), + ); + expect(reads, "exactly one WorkOS read, for the unknown org").toEqual([org]); + expect(result.deleted, "the delete leaves a tombstone behind").toBe("applied"); + expect(result.organization?.name).toBe("Dashboard Org"); + expect(result.tombstone).toMatchObject({ + membershipId: `om_${userId}_${org}`, + status: "inactive", + }); + expect(result.backfilled, "the pre-deletion payload is refused").toBe(false); + expect(result.afterBackfill, "and the member is not live").toBeNull(); + }); + + it("mirrors the organization first when a membership names one the mirror has never seen", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const reads: string[] = []; + const result = await run( + Effect.gen(function* () { + const outcome = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(userId, org)), + ); + const organization = yield* readOrganization(org); + const membership = yield* readMembership(userId, org); + return { outcome, organization, membership }; + }), + stubWorkOS({ + ...profiles(reads), + getOrganization: (id) => { + reads.push(id); + return Effect.succeed(workosOrganization(id, "Dashboard Org")); + }, + }), + ); + expect(reads, "one WorkOS read for the unknown org, one for the unknown member").toEqual([ + org, + userId, + ]); + expect(result.outcome).toBe("applied"); + expect(result.organization?.name).toBe("Dashboard Org"); + expect(result.membership?.membershipId).toBe(`om_${userId}_${org}`); + }); + + it("reads the member's profile from WorkOS when the mirror holds none, never when it does, and mirrors a member WorkOS no longer has bare", async () => { + const org = freshId("org"); + const joiner = freshId("user"); + const known = freshId("user"); + const gone = freshId("user"); + const reads: string[] = []; + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + // `known` signed in before: the stream's own `user.created` for them + // is behind the replay boundary, but the mirror holds their profile. + yield* applyEvent( + userEvent("user.created", workosUser(known, { firstName: "Known", updatedAt: T1 })), + ); + const knownJoins = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(known, org)), + ); + const knownRow = yield* readMembership(known, org); + // `joiner` predates the mirror: no row, no profile event in the + // stream, the org already scanned. The membership event alone + // would leave them nameless. + const joins = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(joiner, org)), + ); + const joinerRow = yield* readMembership(joiner, org); + // A later role change for the now-profiled member reads nothing. + const promoted = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(joiner, org, { role: { slug: "admin" }, updatedAt: T2 }), + ), + ); + // A member WorkOS no longer has (their `user.deleted` is further down + // the stream): mirrored without a profile, the run goes on. + const goneJoins = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(gone, org)), + ); + const goneRow = yield* readMembership(gone, org); + return { knownJoins, knownRow, joins, joinerRow, promoted, goneJoins, goneRow }; + }), + stubWorkOS({ + getUser: (userId) => + Effect.suspend(() => { + reads.push(userId); + return userId === gone + ? Effect.fail(new WorkOSError({ status: 404 })) + : Effect.succeed(workosUser(userId, { firstName: "Fetched" })); + }), + }), + ); + expect(result.knownJoins).toBe("applied"); + expect(result.knownRow?.name, "a profiled member keeps the profile the mirror holds").toBe( + "Known Placeholder", + ); + expect(result.joins).toBe("applied"); + expect(result.joinerRow?.name, "an unprofiled member is mirrored WITH the profile").toBe( + "Fetched Placeholder", + ); + expect(result.joinerRow?.email).toBe(`${joiner}@placeholder.test`); + expect(result.promoted).toBe("applied"); + expect(result.goneJoins, "a member WorkOS no longer has is still mirrored").toBe("applied"); + expect(result.goneRow).toMatchObject({ membershipId: `om_${gone}_${org}`, name: null }); + expect(reads, "one read per unprofiled member, none for a profiled one").toEqual([ + joiner, + gone, + ]); + + // A transient failure reading the profile fails the run (the event is + // retried), exactly as for the organization read. + const blip = await Effect.runPromiseExit( + Effect.exit( + planEvent( + membershipEvent( + "organization_membership.created", + workosMembership(freshId("user"), org), + ), + ), + ).pipe( + Effect.provide( + Layer.mergeAll( + MirrorServices, + stubWorkOS({ getUser: () => Effect.fail(new WorkOSError({ status: 503 })) }), + ), + ), + Effect.scoped, + ), + ); + expect(Exit.isSuccess(blip) && Exit.isFailure(blip.value), "a 5xx keeps the event").toBe(true); + }); + + it("marks the organization deleted for a membership whose organization WorkOS no longer has, but fails on a transient WorkOS failure", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const event = membershipEvent( + "organization_membership.created", + workosMembership(userId, org, { organizationName: "Gone Org" }), + freshId("event"), + T2, + ); + const gone = await run( + Effect.gen(function* () { + const outcome = yield* applyEvent(event); + const organization = yield* readOrganization(org); + const membership = yield* readMembership(userId, org); + // The org's own deletion event, further down the stream, finds the + // mark already there. + const deleted = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Gone Org"), + freshId("event"), + T3, + ), + ); + return { outcome, organization, membership, deleted }; + }), + stubWorkOS({ + getOrganization: () => Effect.fail(new WorkOSError({ status: 404 })), + }), + ); + expect( + gone.outcome, + "an org WorkOS has deleted marks the org deleted, not a failed run and not a dropped event", + ).toBe("applied"); + expect(gone.organization, "a tombstone row is minted for it").toMatchObject({ + name: "Gone Org", + deletedAt: new Date(T2), + }); + expect(gone.membership, "and the membership is not written").toBeNull(); + expect(gone.deleted, "its own deletion event finds the mark").toBe("absent"); + + // A transient failure resolving an org the mirror does not hold (the + // tombstone above would answer the read locally). + const unresolved = membershipEvent( + "organization_membership.created", + workosMembership(userId, freshId("org")), + ); + const blip = await Effect.runPromiseExit( + Effect.exit(planEvent(unresolved)).pipe( + Effect.provide( + Layer.mergeAll( + MirrorServices, + stubWorkOS({ + getOrganization: () => Effect.fail(new WorkOSError({ status: 503 })), + }), + ), + ), + Effect.scoped, + ), + ); + const unreachable = await Effect.runPromiseExit( + Effect.exit(planEvent(unresolved)).pipe( + Effect.provide( + Layer.mergeAll( + MirrorServices, + stubWorkOS({ + getOrganization: () => Effect.fail(new WorkOSError({})), + }), + ), + ), + Effect.scoped, + ), + ); + expect( + Exit.isSuccess(blip) && Exit.isFailure(blip.value), + "a 5xx keeps the event for retry", + ).toBe(true); + expect( + Exit.isSuccess(unreachable) && Exit.isFailure(unreachable.value), + "a network failure keeps the event for retry", + ).toBe(true); + }); + + it("does not create an organization row from organization.updated for an org the mirror has never seen", async () => { + const org = freshId("org"); + const result = await run( + Effect.gen(function* () { + const outcome = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Purged Org")), + ); + const organization = yield* readOrganization(org); + return { outcome, organization }; + }), + ); + expect(result.outcome, "a rename of an unmirrored org is reported absent").toBe("absent"); + expect( + result.organization, + "and mints no row (no resurrection after cloud's purge)", + ).toBeNull(); + }); + + it("mints a tombstone on organization.deleted for an org the mirror has never seen, so a delayed login cannot create it", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + // The org was created, populated, and deleted in the WorkOS + // dashboard before anyone signed in: the mirror has no row for it. + const deleted = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Never Mirrored"), + freshId("event"), + T2, + ), + ); + const tombstone = yield* readOrganization(org); + // A login that fetched its membership list at T1, before the + // deletion, and stalled past it now writes what it holds. + yield* mirrorSignIn( + workosUser(userId), + [workosMembership(userId, org, { organizationName: "Never Mirrored" })], + new Date(T1), + ); + const afterLogin = yield* readOrganization(org); + const membership = yield* readMembership(userId, org, ["active", "pending", "inactive"]); + const replayed = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Never Mirrored"), + freshId("event"), + T3, + ), + ); + return { deleted, tombstone, afterLogin, membership, replayed }; + }), + ); + expect(result.deleted, "the deletion is applied, not dropped for want of a row").toBe( + "applied", + ); + expect(result.tombstone).toMatchObject({ + name: "Never Mirrored", + deletedAt: new Date(T2), + }); + expect(result.tombstone?.slug, "the tombstone is a slugged row like any other").toMatch( + /^never-mirrored/, + ); + expect(result.afterLogin?.deletedAt, "the delayed login does not revive the org").toEqual( + new Date(T2), + ); + expect(result.membership, "nor write the membership").toBeNull(); + expect(result.replayed, "a replayed deletion changes nothing").toBe("absent"); + }); + + it("renames the organization on organization.updated, refuses an older rename, and marks it deleted on organization.deleted, purging nothing", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + const seeded = yield* seedOrganization(org); + yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(userId, org)), + ); + const renamed = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Renamed Org", T2)), + ); + const afterRename = yield* readOrganization(org); + // A rename event older than the name the row holds (replayed, or + // behind a sign-in that already carried the newer name). + const olderRename = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Older Name", T1)), + ); + const afterOlderRename = yield* readOrganization(org); + const deleted = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Renamed Org"), + freshId("event"), + T2, + ), + ); + const orgAfterDelete = yield* readOrganization(org); + const membershipAfterDelete = yield* readMembership(userId, org); + const deletedAgain = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Renamed Org"), + freshId("event"), + T3, + ), + ); + const renamedAfterDelete = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Late Rename", T3)), + ); + const orgAfterReplay = yield* readOrganization(org); + return { + seeded, + renamed, + afterRename, + olderRename, + afterOlderRename, + deleted, + orgAfterDelete, + membershipAfterDelete, + deletedAgain, + renamedAfterDelete, + orgAfterReplay, + }; + }), + stubWorkOS(profiles()), + ); + expect(result.renamed).toBe("applied"); + expect(result.afterRename?.name).toBe("Renamed Org"); + expect(result.afterRename?.slug, "the slug is stable across renames").toBe(result.seeded.slug); + expect(result.olderRename, "an older rename is refused").toBe("stale"); + expect(result.afterOlderRename?.name).toBe("Renamed Org"); + expect(result.deleted, "organization.deleted marks the org").toBe("applied"); + expect(result.orgAfterDelete?.deletedAt, "as of the event").toEqual(new Date(T2)); + expect(result.orgAfterDelete?.name, "the row is kept, not purged").toBe("Renamed Org"); + expect(result.membershipAfterDelete, "and so is the membership row").not.toBeNull(); + expect(result.deletedAgain, "a replayed deletion changes nothing").toBe("absent"); + expect(result.renamedAfterDelete, "a deleted org is never renamed").toBe("absent"); + expect(result.orgAfterReplay?.deletedAt, "the first mark stands").toEqual(new Date(T2)); + expect(result.orgAfterReplay?.name).toBe("Renamed Org"); + }); +}); + +describe("syncWorkOsEvents", () => { + /** Pin the instance-wide cursor to a fresh known value, whatever it was. */ + const pinCursor = (value: string) => + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const before = yield* mirror.getCursor(); + const moved = yield* mirror.applyPage(before, value, []); + expect(Option.isSome(moved)).toBe(true); + return value; + }); + + type Page = { + readonly data: readonly WorkOsMirroredEvent[]; + readonly after: string | null; + }; + + /** + * A fake Events API serving `pages` in order, recording every request's + * paging options; `onPage` runs before the nth page is returned (the CAS + * contention test moves the cursor from there). `methods` adds any other + * WorkOS call the run under test is allowed to make. + */ + const eventsApi = ( + pages: readonly Page[], + requests: WorkOSListEventsOptions[], + onPage: (index: number) => Effect.Effect = () => Effect.void, + methods: Partial = {}, + ) => + Effect.map(WorkOsMirror.asEffect(), (mirror) => + stubWorkOS({ + ...methods, + listEvents: (options) => + Effect.gen(function* () { + const index = requests.length; + requests.push(options); + yield* onPage(index).pipe(Effect.provideService(WorkOsMirror, mirror), Effect.orDie); + const page = pages[index] ?? { data: [], after: null }; + return { + object: "list" as const, + data: [...page.data], + listMetadata: { before: null, after: page.after }, + }; + }), + }), + ); + + const sync = ( + workos: Layer.Layer, + ): Effect.Effect => + syncWorkOsEvents().pipe(Effect.provide(workos)); + + it("pages from the persisted cursor, applies every event, and commits the last id of each page", async () => { + const org = freshId("org"); + const a = freshId("user"); + const b = freshId("user"); + const requests: WorkOSListEventsOptions[] = []; + const profileReads: string[] = []; + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + const start = yield* pinCursor(freshId("event")); + const startedAt = Date.now(); + const pages: Page[] = [ + { + data: [ + userEvent("user.created", workosUser(a), `${start}_1`), + membershipEvent( + "organization_membership.created", + workosMembership(a, org), + `${start}_2`, + ), + ], + after: `${start}_2`, + }, + { + data: [ + membershipEvent( + "organization_membership.created", + workosMembership(b, org), + `${start}_3`, + ), + organizationEvent( + "organization.deleted", + workosOrganization(org, "Gone"), + `${start}_4`, + ), + ], + after: null, + }, + ]; + const workos = yield* eventsApi(pages, requests, () => Effect.void, profiles(profileReads)); + const report = yield* sync(workos); + const mirror = yield* WorkOsMirror; + const cursor = yield* mirror.getCursor(); + const members = yield* Effect.flatMap(MemberDirectory.asEffect(), (d) => d.members(org)); + const drainedAt = yield* mirror.drainedAt(); + return { start, startedAt, report, cursor, members, drainedAt }; + }), + ); + expect(requests.map((r) => r.after)).toEqual([result.start, `${result.start}_2`]); + expect(requests[0]).toMatchObject({ order: "asc", limit: 100 }); + expect(requests[0]?.rangeStart, "a run with a cursor never sends rangeStart").toBeUndefined(); + expect(result.report).toMatchObject({ + pages: 2, + events: 4, + applied: 4, + stopped: "drained", + cursor: `${result.start}_4`, + }); + expect(result.cursor).toBe(`${result.start}_4`); + expect(result.members.map((m) => m.accountId).sort()).toEqual([a, b].sort()); + expect( + profileReads, + "only the member whose profile the stream did not carry is read from WorkOS", + ).toEqual([b]); + expect( + result.drainedAt, + "a run that reads the stream to its end records the drain", + ).not.toBeNull(); + expect( + result.drainedAt!.getTime(), + "as of the run's start, so it never post-dates an event the run did not see", + ).toBeGreaterThanOrEqual(result.startedAt - 1000); + expect(result.drainedAt!.getTime()).toBeLessThanOrEqual(Date.now()); + }); + + /** The sync row is instance-wide: clear it so the run under test is a first run. */ + const clearSyncRow = Effect.flatMap(DbService.asEffect(), ({ db }) => + Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)), + ); + + it("starts from the backfill's replay boundary when no cursor exists", async () => { + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + yield* clearSyncRow; + yield* mirror.setReplayBoundary(new Date(T2)); + const start = freshId("event"); + const workos = yield* eventsApi( + [ + { + data: [userEvent("user.created", workosUser(freshId("user")), start)], + after: null, + }, + ], + requests, + ); + const report = yield* sync(workos); + const cursor = yield* mirror.getCursor(); + return { report, cursor, start }; + }), + ); + expect(requests).toHaveLength(1); + expect(requests[0]?.after).toBeUndefined(); + expect( + requests[0]?.rangeStart, + "the first read starts exactly where the backfill began reading WorkOS", + ).toBe(T2); + expect(result.report.stopped).toBe("drained"); + expect(result.cursor, "the first run mints the cursor").toBe(result.start); + }); + + it("reads nothing while neither a cursor nor a replay boundary exists", async () => { + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + yield* clearSyncRow; + const workos = yield* eventsApi( + [ + { + data: [userEvent("user.created", workosUser(freshId("user")))], + after: null, + }, + ], + requests, + ); + const report = yield* sync(workos); + const cursor = yield* mirror.getCursor(); + const drainedAt = yield* mirror.drainedAt(); + return { report, cursor, drainedAt }; + }), + ); + expect(requests, "no wall-clock guess is ever sent to WorkOS").toHaveLength(0); + expect(result.report).toMatchObject({ + pages: 0, + events: 0, + stopped: "awaiting_backfill", + }); + expect(result.cursor, "and no cursor is minted").toBeNull(); + expect(result.drainedAt, "nor is a drain recorded: nothing was read").toBeNull(); + }); + + it("stops when another run moves the cursor under it, writing nothing from the contended page", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + const mirror = yield* WorkOsMirror; + const start = yield* pinCursor(freshId("event")); + const intruder = `${start}_intruder`; + // This run's page carries a membership update that the leading run + // has already applied AND deleted (the member was revoked). If the + // lagging run's page landed, the revoked member would be back. + const pages: Page[] = [ + { + data: [ + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { role: { slug: "admin" } }), + `${start}_1`, + ), + ], + after: `${start}_1`, + }, + { + data: [userEvent("user.created", workosUser(freshId("user")), `${start}_2`)], + after: null, + }, + ]; + // While this run is reading its first page, "another run" applies the + // same page, then the membership's deletion, and commits both. + const workos = yield* eventsApi( + pages, + requests, + (index) => + index === 0 + ? Effect.gen(function* () { + const leading = yield* WorkOsMirror; + yield* leading.applyPage(start, `${start}_1`, [ + WorkOsMirrorWrite.UpsertMembership({ + membership: mirrorMembershipFromWorkOs(workosMembership(userId, org)), + }), + ]); + yield* leading.applyPage(`${start}_1`, intruder, [ + WorkOsMirrorWrite.DeleteMembership({ + membership: { + id: `om_${userId}_${org}`, + accountId: userId, + organizationId: org, + }, + deletedAt: new Date(T2), + }), + ]); + }) + : Effect.void, + profiles(), + ); + const drainedBefore = yield* mirror.drainedAt(); + const report = yield* sync(workos); + const cursor = yield* mirror.getCursor(); + const membership = yield* readMembership(userId, org); + const drainedAfter = yield* mirror.drainedAt(); + return { report, cursor, intruder, membership, drainedBefore, drainedAfter }; + }), + ); + expect(requests, "the second page is never read").toHaveLength(1); + expect(result.report).toMatchObject({ + pages: 1, + events: 1, + applied: 0, + stopped: "cursor_contended", + }); + expect(result.cursor, "the other run's cursor stands").toBe(result.intruder); + expect(result.membership, "the revoked membership is not resurrected").toBeNull(); + expect( + result.drainedAfter, + "a run that yielded the stream drained nothing and records no drain", + ).toEqual(result.drainedBefore); + }); + + it("advances the cursor past a membership event whose organization WorkOS no longer has, marking the org deleted", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + const start = yield* pinCursor(freshId("event")); + const workos = yield* eventsApi( + [ + { + data: [ + membershipEvent( + "organization_membership.created", + workosMembership(userId, org), + `${start}_1`, + ), + organizationEvent( + "organization.deleted", + workosOrganization(org, "Gone"), + `${start}_2`, + ), + ], + after: null, + }, + ], + requests, + () => Effect.void, + { + getOrganization: () => Effect.fail(new WorkOSError({ status: 404 })), + }, + ); + const report = yield* sync(workos); + const mirror = yield* WorkOsMirror; + const cursor = yield* mirror.getCursor(); + const organization = yield* readOrganization(org); + return { start, report, cursor, organization }; + }), + ); + expect(result.report).toMatchObject({ + pages: 1, + events: 2, + applied: 1, + absent: 1, + stopped: "drained", + cursor: `${result.start}_2`, + }); + expect(result.cursor, "the stream is not stalled on the gone org").toBe(`${result.start}_2`); + expect(result.organization?.deletedAt, "the org is left as a tombstone").not.toBeNull(); + }); +}); + +describe("workos webhook", () => { + const SECRET = "whsec_placeholder_signing_secret"; + + const handlerFor = (deps: { + readonly secret: string | undefined; + readonly detached: Promise[]; + readonly synced: number[]; + }) => + HttpRouter.toWebHandler( + makeWorkOsWebhookRoute({ + secret: deps.secret, + detach: (work) => { + deps.detached.push(work); + }, + sync: () => { + deps.synced.push(1); + return Promise.resolve(); + }, + }).pipe( + // The REAL client: its `webhooks.constructEvent` is the signature check + // under test. The api key / client id it reads are the vitest env's. + Layer.provideMerge(WorkOSClient.Default), + Layer.provideMerge(HttpServer.layerServices), + ), + { disableLogger: true }, + ).handler; + + const delivery = { + id: "event_placeholder", + event: "user.created", + created_at: T1, + context: {}, + data: { + object: "user", + id: "user_placeholder", + email: "member@placeholder.test", + email_verified: true, + first_name: "Ada", + last_name: "Placeholder", + profile_picture_url: null, + last_sign_in_at: T1, + locale: null, + created_at: T1, + updated_at: T1, + external_id: null, + metadata: {}, + }, + }; + + /** The `WorkOS-Signature` header WorkOS sends: `t=, v1=`. */ + const signature = (body: string, secret: string, timestamp = Date.now()) => + `t=${timestamp}, v1=${createHmac("sha256", secret).update(`${timestamp}.${body}`).digest("hex")}`; + + const post = (body: string, headers: Record) => + new Request(`http://test.local${WORKOS_WEBHOOK_PATH}`, { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body, + }); + + const deps = (secret: string | undefined) => ({ + secret, + detached: [] as Promise[], + synced: [] as number[], + }); + + it("accepts a genuinely signed delivery and pokes the reconciler past the response", async () => { + const d = deps(SECRET); + const body = JSON.stringify(delivery); + const response = await handlerFor(d)( + post(body, { "workos-signature": signature(body, SECRET) }), + ); + expect(response.status).toBe(200); + expect(d.synced, "one reconciler pass").toHaveLength(1); + expect(d.detached, "handed to the platform, not awaited in the response").toHaveLength(1); + }); + + it("rejects a delivery signed with another secret, a tampered body, and a missing header", async () => { + const d = deps(SECRET); + const handler = handlerFor(d); + const body = JSON.stringify(delivery); + + const wrongSecret = await handler( + post(body, { "workos-signature": signature(body, "whsec_other") }), + ); + const tampered = await handler( + post(body.replace("user_placeholder", "user_tampered"), { + "workos-signature": signature(body, SECRET), + }), + ); + const unsigned = await handler(post(body, {})); + const notJson = await handler( + post("not json", { "workos-signature": signature("not json", SECRET) }), + ); + const expired = await handler( + post(body, { + "workos-signature": signature(body, SECRET, Date.now() - 10 * 60 * 1000), + }), + ); + + expect([ + wrongSecret.status, + tampered.status, + unsigned.status, + notJson.status, + expired.status, + ]).toEqual([400, 400, 400, 400, 400]); + expect(d.synced, "nothing is poked").toEqual([]); + }); + + it("refuses every delivery while no signing secret is configured", async () => { + const d = deps(undefined); + const body = JSON.stringify(delivery); + const response = await handlerFor(d)( + post(body, { "workos-signature": signature(body, SECRET) }), + ); + expect(response.status).toBe(503); + expect(d.synced).toEqual([]); + }); +}); diff --git a/apps/cloud/src/auth/workos-events-sync.ts b/apps/cloud/src/auth/workos-events-sync.ts new file mode 100644 index 000000000..d404d5fbf --- /dev/null +++ b/apps/cloud/src/auth/workos-events-sync.ts @@ -0,0 +1,107 @@ +// --------------------------------------------------------------------------- +// The membership mirror's reconciler, bound to the Worker's services: the +// replay itself is `workos-events-replay.ts` (a pure function over its +// ports, so the deploy gate can run the same replay under bun); this file +// wires those ports to `WorkOSClient`, `UserStoreService`, and +// `WorkOsMirror` for the every-minute cron and the signed webhook poke +// (`workos-events-runner.ts`). +// +// The only translation here is the WorkOS answer "gone": the replay wants +// `None` for an organization or user WorkOS no longer has, and only a 404 +// says that. A 401/403 is a credentials or permissions problem with THIS +// deployment, and 429/5xx/no status is a blip: all of those stay failures +// so the run stops and the event is retried once fixed, not skipped and +// lost. +// --------------------------------------------------------------------------- + +import { Effect, Option } from "effect"; + +import { UserStoreService } from "./context"; +import type { UserStoreError, WorkOSError } from "./errors"; +import { WorkOSClient } from "./workos"; +import { + planWorkOsEvent, + replayWorkOsEvents, + type PlannedProfiles, + type WorkOsEventsReplayDeps, + type WorkOsMirroredEvent, +} from "./workos-events-replay"; +import { WorkOsMirror, type WorkOsMirrorError } from "./workos-mirror"; + +export { + MIRRORED_EVENT_NAMES, + isMirroredEvent, + type PlannedProfiles, + type WorkOsEventOutcome, + type WorkOsEventsSyncReport, + type WorkOsMirroredEvent, + type WorkOsMirroredEventName, +} from "./workos-events-replay"; + +type SyncFailure = WorkOSError | UserStoreError | WorkOsMirrorError; + +// A 404 is the deterministic "gone" the replay acts on; everything else +// fails the run (see the header). +const noneWhenGone = ( + read: Effect.Effect, +): Effect.Effect, WorkOSError> => + read.pipe( + Effect.map(Option.some), + Effect.catchTag("WorkOSError", (error) => + error.status === 404 ? Effect.succeed(Option.none()) : Effect.fail(error), + ), + ); + +const replayDeps: Effect.Effect< + WorkOsEventsReplayDeps, + never, + WorkOSClient | UserStoreService | WorkOsMirror +> = Effect.gen(function* () { + const workos = yield* WorkOSClient; + const users = yield* UserStoreService; + const mirror = yield* WorkOsMirror; + return { + source: { + listEvents: (options) => + Effect.map(workos.listEvents(options), (page) => ({ + data: page.data, + after: page.listMetadata.after ?? null, + })), + getOrganization: (organizationId) => noneWhenGone(workos.getOrganization(organizationId)), + getUser: (userId) => noneWhenGone(workos.getUser(userId)), + }, + store: { + getOrganization: (organizationId) => + users.use("getOrganization", (s) => s.getOrganization(organizationId)), + upsertOrganization: (organization) => + users.use("upsertOrganization", (s) => s.upsertOrganization(organization)), + getAccount: (accountId) => users.use("getAccount", (s) => s.getAccount(accountId)), + }, + mirror, + }; +}); + +/** + * Translate one event into the mirror write it calls for, over the Worker's + * services. See `planWorkOsEvent` for the contract. + */ +export const planEvent = Effect.fn("workos_events.plan")(function* ( + event: WorkOsMirroredEvent, + profiled: PlannedProfiles = new Set(), +) { + yield* Effect.annotateCurrentSpan({ + "workos.event": event.event, + "workos.event_id": event.id, + }); + const deps = yield* replayDeps; + return yield* planWorkOsEvent(deps, event, profiled); +}); + +/** + * One reconciler run over the Worker's services. See `replayWorkOsEvents` + * for the contract. + */ +export const syncWorkOsEvents = Effect.fn("workos_events.sync")(function* () { + const deps = yield* replayDeps; + return yield* replayWorkOsEvents(deps); +}); diff --git a/apps/cloud/src/auth/workos-mirror-backfill.ts b/apps/cloud/src/auth/workos-mirror-backfill.ts index adcd7c11f..d6875241b 100644 --- a/apps/cloud/src/auth/workos-mirror-backfill.ts +++ b/apps/cloud/src/auth/workos-mirror-backfill.ts @@ -42,17 +42,18 @@ // caller's own membership) starts unmarked and is scanned before any count // read from the mirror is trusted. A run over every organization // (`backfillWorkOsMirror`) additionally records the Events API replay -// boundary — the instant it began reading WorkOS, taken BEFORE anything is -// listed so no change can fall between the boundary and a listing — once -// every organization was written, and only if no boundary is recorded yet. -// A run that fails part-way records nothing, and a later completed run -// keeps the first boundary: a scan refreshes memberships and tombstones, -// not organization names or deleted users' profiles, so an -// `organization.updated` or `user.deleted` between two runs is covered only -// by the events stream — advancing the boundary past it would skip it for -// good. The reconciler's own cursor takes over from the boundary after its -// first page, so the boundary's only job is to name where that first page -// starts. +// boundary — the instant it began reading WorkOS — BEFORE anything is +// listed, and only if no boundary is recorded yet. Recording it first, not +// on completion, is what makes a failed run safe to retry: a run that +// fails part-way has already fixed the boundary at its start, and its +// retry reads that boundary back instead of taking a fresh, later one. A +// scan refreshes memberships and tombstones, not organization names or +// deleted users' profiles, so an `organization.updated` or `user.deleted` +// that lands between the attempts (or between two runs) is covered only by +// the events stream — a boundary taken by the retry would fall after it and +// skip it for good, leaving the deleted user's profile in the mirror. The +// reconciler's own cursor takes over from the boundary after its first +// page, so the boundary's only job is to name where that first page starts. // // The mark also orders every OTHER membership write against the scan: the // mirror refuses a membership payload stamped before the organization's @@ -196,11 +197,12 @@ export const backfillOrganization = ( ); /** - * Run the full backfill: scan every organization the mirror knows, then - * record the Events API replay boundary if none is recorded yet. Fails on - * the first source or mirror failure — the organizations scanned so far stay - * marked (each was covered in full), the boundary is not recorded, and the - * run is safe to repeat. + * Run the full backfill: record the Events API replay boundary if none is + * recorded yet, then scan every organization the mirror knows. Fails on the + * first source or mirror failure — the organizations scanned so far stay + * marked (each was covered in full), the boundary recorded at the start + * stands, and the run is safe to repeat: the retry keeps that boundary, so + * every change since the first attempt began is the reconciler's to replay. */ export const backfillWorkOsMirror = ( source: WorkOsMirrorBackfillSource, @@ -208,8 +210,21 @@ export const backfillWorkOsMirror = ( options: WorkOsMirrorBackfillOptions, ) => Effect.gen(function* () { - // Taken before the first listing; recorded only once the run completes. + // The replay boundary: taken AND recorded before anything is listed, so + // every change from this instant on is the events stream's to apply — + // one that lands while this run is still listing, or between this run + // failing part-way and its retry. Kept only when none is recorded yet + // (`setReplayBoundary`): a retry or a later run reads the first one + // back instead of moving it. A dry run records nothing. const boundary = yield* now(); + if (!options.dryRun) { + const recorded = yield* mirror.setReplayBoundary(boundary); + options.log( + recorded + ? `events replay boundary set to ${boundary.toISOString()}` + : "events replay boundary already recorded by an earlier run; kept (the events reconciler replays every change since it)", + ); + } const organizationIds = yield* source.listOrganizationIds(); let memberships = 0; @@ -246,22 +261,11 @@ export const backfillWorkOsMirror = ( : `${counts.organizations} organization(s), ${counts.memberships} membership(s): wrote ${counts.usersWritten} user(s), ${counts.membershipsWritten} membership(s), tombstoned ${counts.membershipsTombstoned}`, ); if (!options.dryRun) { - // Every organization was read and written without failure (a failure - // above fails the whole run) — or refused in favour of a listing taken - // later still — so everything before `boundary` is now covered: record - // it for the reconciler — unless an earlier run already did, in which - // case the events between the two runs are the reconciler's to replay - // and the earlier boundary stands. - const recorded = yield* mirror.setReplayBoundary(boundary); - options.log( - recorded - ? `events replay boundary set to ${boundary.toISOString()}` - : "events replay boundary already recorded by an earlier run; kept (the events reconciler replays every change since it)", - ); - // And every live organization is now covered: the mirror is complete - // enough to authorize from (once the reconciler has caught up too), - // which the authorization path reads as the first half of readiness. - // Once: a re-run keeps the first completion. + // Every live organization is now covered (a failure above fails the + // whole run): the mirror is complete enough to authorize from, once + // the reconciler has caught up too — the first half of the readiness + // the authorization path checks. Once: a re-run keeps the first + // completion. const completedAt = yield* now(); const marked = yield* mirror.markBackfillCompleted(completedAt); options.log( diff --git a/apps/cloud/src/auth/workos-mirror-store.ts b/apps/cloud/src/auth/workos-mirror-store.ts index e9c7e544b..1a5886283 100644 --- a/apps/cloud/src/auth/workos-mirror-store.ts +++ b/apps/cloud/src/auth/workos-mirror-store.ts @@ -45,14 +45,17 @@ // cleared, stamped with the deletion), and no membership naming that account // is written again, however the payload is stamped — WorkOS never reuses a // user id, and a membership the mirror had not seen has no row of its own -// for a guard to refuse the insert against. The cursor advances only by -// compare-and-set, so two reconciler runs cannot both own the stream. +// for a guard to refuse the insert against. The reconciler applies a page +// of events and advances the cursor in ONE transaction that +// compare-and-sets the cursor first (`applyPage`), so a run that has lost +// the stream to another run writes nothing — the `updatedAt` guard alone +// cannot stop it re-applying a page the leading run has moved past. // // A backfill SCAN of one organization (its full membership listing, taken -// at one instant) is applied in ONE transaction that first compare-and-sets -// the organization's `backfilled_at` to the listing's instant -// (`applyOrganizationScan`): the row stays locked until commit, so two -// overlapping scans serialize on it, and the one whose listing is older +// at one instant) is applied the same way: ONE transaction that first +// compare-and-sets the organization's `backfilled_at` to the listing's +// instant (`applyOrganizationScan`), so the row stays locked until commit, +// two overlapping scans serialize on it, and the one whose listing is older // than the recorded one writes NOTHING. The `updatedAt` guard alone cannot // order two scans: a scan that listed a membership, stalled, and resumed // after a later scan had found it gone would insert it live — the later scan @@ -72,15 +75,16 @@ // listing that sets the mark. // // The events replay boundary (`workos_sync.range_start`) is written ONCE, by -// the first completed backfill, and never advanced: a later backfill -// refreshes memberships only, so an organization rename or user deletion -// between two runs is covered by the events stream alone, and moving the -// boundary past it would skip it for good. +// the first backfill run BEFORE its first listing, and never advanced: a +// scan refreshes memberships only, so an organization rename or user +// deletion after that instant — between two runs, or between a run that +// failed part-way and its retry — is covered by the events stream alone, +// and moving the boundary past it would skip it for good. // --------------------------------------------------------------------------- import { and, eq, isNotNull, isNull, lt, ne, notInArray, or, sql } from "drizzle-orm"; import type { AnyPgColumn } from "drizzle-orm/pg-core"; -import { Effect, Option } from "effect"; +import { Data, Effect, Option } from "effect"; import type { MemberStatus } from "@executor-js/api/server"; @@ -92,6 +96,7 @@ import { workosSync, } from "../db/schema"; import type { DrizzleDb } from "../db/db"; +import { insertOrganization, organizationAcceptsName } from "./user-store"; import { WorkOsMirrorError, tryPromiseService, @@ -163,6 +168,70 @@ export interface WorkOsOrganizationScanWrites { readonly membershipsTombstoned: number; } +/** + * One write of a reconciler page, applied by `applyPage` inside the page's + * transaction. The reconciler plans a page into these BEFORE the transaction + * opens, so every WorkOS read (resolving an organization the mirror has never + * seen) is done by then: the transaction holds the mirror's single connection + * and must not wait on the network. + */ +export type WorkOsMirrorWrite = Data.TaggedEnum<{ + readonly UpsertUser: { readonly user: WorkOsMirrorUser }; + readonly UpsertMembership: { readonly membership: WorkOsMirrorMembership }; + /** + * A membership together with its member's profile, read from WorkOS at + * plan time because the mirror held no profile for the account + * (`workos-events-sync.ts`): the user is upserted first, under the usual + * guard, then the membership. The outcome is the membership's. + */ + readonly UpsertMember: { + readonly user: WorkOsMirrorUser; + readonly membership: WorkOsMirrorMembership; + }; + /** Tombstone a membership as of `deletedAt` (the event's `createdAt`). */ + readonly DeleteMembership: { + readonly membership: WorkOsMirrorMembershipRef; + readonly deletedAt: Date; + }; + /** Tombstone a user and their memberships as of `deletedAt`. */ + readonly DeleteUser: { readonly accountId: string; readonly deletedAt: Date }; + /** + * Rename an organization the mirror already holds — never inserts one — + * from a payload stamped `updatedAt` (the WorkOS organization's own), under + * the same name guard every feeder applies (`organizationAcceptsName`). + */ + readonly RenameOrganization: { + readonly organizationId: string; + readonly name: string; + readonly updatedAt: Date; + }; + /** + * Mark an organization deleted as of `deletedAt` (the event's + * `createdAt`) — minting the row as a TOMBSTONE, named `name`, when the + * mirror has never seen the organization, so a feeder still holding a + * membership of it (a login that stalled across the deletion) finds the + * tombstone and cannot mint the organization live. Never purges tenant + * data (that is cloud's own flow, `db/org-deletion.ts`). An earlier mark + * stands (`absent`). + */ + readonly MarkOrganizationDeleted: { + readonly organizationId: string; + readonly name: string; + readonly deletedAt: Date; + }; +}>; +export const WorkOsMirrorWrite = Data.taggedEnum(); + +/** + * What one write did: a row was written, renamed, or tombstoned (`applied`); + * the `updatedAt` guard refused an older payload (`stale`); or a delete found + * its row already tombstoned or superseded by a newer membership (a + * replayed delete), or a rename found no live organization row to change — + * the mirror has never seen it, or it is marked deleted — or a deletion + * mark found the organization already marked (`absent`). + */ +export type WorkOsMirrorWriteOutcome = "applied" | "stale" | "absent"; + export interface WorkOsMirrorShape { /** * Insert or refresh a user row. `false` when the payload was refused and @@ -250,14 +319,20 @@ export interface WorkOsMirrorShape { /** The id of the last WorkOS event applied, or `null` before the first run. */ readonly getCursor: () => Effect.Effect; /** - * Compare-and-set the cursor: advance to `next` only if it still reads - * `prev` (`null` = no cursor yet). `false` means another run moved it first - * — the caller must stop, it no longer owns the stream. + * Apply one reconciler page atomically: in a single transaction, + * compare-and-set the cursor from `prev` (`null` = no cursor yet) to + * `next`, and only if that succeeded apply `writes` in order. The cursor + * row stays locked until commit, so two runs applying pages serialize on + * it and the one whose `prev` is stale sees the moved cursor and writes + * nothing: `None` means another run owns the stream and the caller must + * stop. `Some` carries one outcome per write, in order. An empty `writes` + * is a bare cursor advance. */ - readonly setCursor: ( + readonly applyPage: ( prev: string | null, next: string, - ) => Effect.Effect; + writes: readonly WorkOsMirrorWrite[], + ) => Effect.Effect, WorkOsMirrorError>; /** * Apply one organization's backfill scan — the memberships (with their * users) a WorkOS listing taken at `listedAt` contained — atomically: in a @@ -283,24 +358,25 @@ export interface WorkOsMirrorShape { scan: WorkOsOrganizationScan, ) => Effect.Effect, WorkOsMirrorError>; /** - * The Events API replay boundary: the instant the FIRST completed one-off - * backfill began reading WorkOS, or `null` if none has completed. The + * The Events API replay boundary: the instant the FIRST one-off backfill + * run began reading WorkOS, or `null` if none has started. The * reconciler's first run (no cursor yet) reads the stream from here — the * backfill covers everything before it — and without a boundary it must * not guess. */ readonly replayBoundary: () => Effect.Effect; /** - * Record the replay boundary, ONCE: `at` is the instant a completed - * backfill began reading WorkOS, and it is kept only when no boundary is - * recorded yet — `true` when this call recorded it. A later completed run - * never moves it: the backfill refreshes memberships and tombstones only, - * not organization names or deleted users' profiles, so a change between - * two runs is covered only by the events stream, which must still be read - * from the first boundary. Written only after every organization has been - * written, so a run that fails part-way records nothing. Never touches the - * cursor: a stream already being followed keeps its position, and the - * boundary is then unused. + * Record the replay boundary, ONCE: `at` is the instant a backfill run + * began reading WorkOS, and it is kept only when no boundary is recorded + * yet — `true` when this call recorded it. A later run never moves it: the + * backfill refreshes memberships and tombstones only, not organization + * names or deleted users' profiles, so a change after the first boundary + * is covered only by the events stream, which must still be read from + * there. Written BEFORE the run's first listing, so a run that fails + * part-way leaves the boundary standing and its retry keeps it — a + * `user.deleted` between the attempts stays inside the replay. Never + * touches the cursor: a stream already being followed keeps its position, + * and the boundary is then unused. */ readonly setReplayBoundary: (at: Date) => Effect.Effect; /** @@ -323,6 +399,22 @@ export interface WorkOsMirrorShape { * boundary. */ readonly markBackfillCompleted: (at: Date) => Effect.Effect; + /** + * When a reconciler run last read the events stream to its end + * (`workos_sync.drained_at`), or `null` if none has. The second half of + * the mirror's readiness for authorization: a mirror whose reconciler + * has not caught up within the lag budget may still hold a membership + * WorkOS has since revoked. + */ + readonly drainedAt: () => Effect.Effect; + /** + * Record that a reconciler run read the stream to its end at `at`. Moves + * the mark forward only — a run that finished after a later one keeps the + * later mark — and only on the row a run already owns: the events row is + * minted by the boundary or the first cursor advance, so a missing row + * means nothing was drained and nothing is written (`false`). + */ + readonly markDrained: (at: Date) => Effect.Effect; /** * When the organization's membership list was last FULLY scanned from * WorkOS (`backfillOrganization` in workos-mirror-backfill.ts), or `null` @@ -502,9 +594,9 @@ const membershipDeletableBy = (id: string) => and(or(isNull(memberships.membershipId), eq(memberships.membershipId, id)), notDeleted); // The write queries, over `db` or over a transaction handle (drizzle's is a -// `PgDatabase` too): the one `applyOrganizationScan` opens, or the one the -// store opens per `upsertMembership`. Each answers whether it wrote a row; -// the public shape translates that. +// `PgDatabase` too): the one `applyPage` or `applyOrganizationScan` opens, +// or the one the store opens per `upsertMembership`. Each answers whether +// it wrote a row; the public shape and the transactions translate that. const makeWrites = (db: DrizzleDb) => { const ensureAccount = (id: string) => db.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id }); @@ -742,6 +834,82 @@ const makeWrites = (db: DrizzleDb) => { return cleared.length > 0; }, + // An UPDATE, never an insert: the slug is minted only by + // `upsertOrganization` (auth/user-store.ts), and an org purged by cloud's + // own deletion flow must not come back — with a fresh slug and no members + // — because a rename that preceded the deletion is replayed after it. + // Only of a LIVE row, and only from a payload at least as new as the one + // that last named it: the same guard `upsertOrganization` applies, so + // an event rename and a sign-in's name (stamped by its fetch) order + // each other however they arrive. + renameOrganization: async ( + organizationId: string, + name: string, + updatedAt: Date, + ): Promise => { + const renamed = await db + .update(organizations) + .set({ name, workosUpdatedAt: updatedAt }) + .where( + and( + eq(organizations.id, organizationId), + isNull(organizations.deletedAt), + organizationAcceptsName(updatedAt), + ), + ) + .returning({ id: organizations.id }); + if (renamed.length > 0) return "applied"; + // Refused: tell a live row the guard held back (`stale`) from a row + // the mirror does not hold or holds as deleted (`absent`). + const live = await db + .select({ id: organizations.id }) + .from(organizations) + .where(and(eq(organizations.id, organizationId), isNull(organizations.deletedAt))); + return live.length > 0 ? "stale" : "absent"; + }, + + // Marks a live row, or MINTS a tombstone row when the mirror has never + // seen the organization: an org created, populated, and deleted in the + // WorkOS dashboard before anyone signed in leaves no row behind + // otherwise, and a login that fetched its memberships before the + // deletion (and stalled) would then mint the org live, with nothing + // left in the stream to revoke it — this event is consumed. A row + // already marked is left alone: cloud's own deletion flow marks the org + // before deleting it in WorkOS, so the event that follows finds the + // mark already there and changes nothing — `false`, as for a replayed + // event. Minted through the one slug mint point (`insertOrganization`), + // so a tombstone is a routable, unique-slugged row like any other; the + // mark alone is what refuses it. + markOrganizationDeleted: async ( + organizationId: string, + name: string, + deletedAt: Date, + ): Promise => { + const mark = async () => { + const marked = await db + .update(organizations) + .set({ deletedAt }) + .where(and(eq(organizations.id, organizationId), isNull(organizations.deletedAt))) + .returning({ id: organizations.id }); + return marked.length > 0; + }; + if (await mark()) return true; + const held = await db + .select({ id: organizations.id }) + .from(organizations) + .where(eq(organizations.id, organizationId)); + if (held.length > 0) return false; + const minted = await insertOrganization(db, { + id: organizationId, + name, + workosUpdatedAt: null, + deletedAt, + }); + // A concurrent feeder may have minted the row LIVE between the read + // and the insert (the insert then yields its row): mark that one. + return minted.deletedAt !== null || mark(); + }, + // Tombstone (at `listedAt`) every membership of the organization that a // listing taken at `listedAt` did not contain. Only inside a scan's // transaction, after its `backfilled_at` CAS: on its own this could @@ -800,6 +968,57 @@ const makeWrites = (db: DrizzleDb) => { }; }; +type Writes = ReturnType; + +const applyWrite = (writes: Writes, write: WorkOsMirrorWrite): Promise => + WorkOsMirrorWrite.$match(write, { + UpsertUser: async ({ user }) => ((await writes.upsertUser(user)) ? "applied" : "stale"), + UpsertMembership: async ({ membership }) => + (await writes.upsertMembership(membership)) ? "applied" : "stale", + UpsertMember: async ({ user, membership }) => { + await writes.upsertUser(user); + return (await writes.upsertMembership(membership)) ? "applied" : "stale"; + }, + DeleteMembership: async ({ membership, deletedAt }) => + (await writes.deleteMembership(membership, deletedAt)) ? "applied" : "absent", + DeleteUser: async ({ accountId, deletedAt }) => + (await writes.deleteUser(accountId, deletedAt)) ? "applied" : "absent", + RenameOrganization: ({ organizationId, name, updatedAt }) => + writes.renameOrganization(organizationId, name, updatedAt), + MarkOrganizationDeleted: async ({ organizationId, name, deletedAt }) => + (await writes.markOrganizationDeleted(organizationId, name, deletedAt)) + ? "applied" + : "absent", + }); + +// Compare-and-set the events cursor. Run inside a transaction this also +// LOCKS the cursor row until commit: a concurrent run's CAS waits here, then +// re-reads the moved cursor and matches nothing. +const advanceCursor = async (db: DrizzleDb, prev: string | null, next: string) => { + const now = new Date(); + if (prev === null) { + // First advance: mint the row, or claim an existing row that still has + // no cursor. A row that already carries one belongs to another run and + // is left alone. + const written = await db + .insert(workosSync) + .values({ id: WORKOS_EVENTS_STREAM_ID, cursor: next, updatedAt: now }) + .onConflictDoUpdate({ + target: workosSync.id, + set: { cursor: next, updatedAt: now }, + setWhere: isNull(workosSync.cursor), + }) + .returning({ id: workosSync.id }); + return written.length > 0; + } + const written = await db + .update(workosSync) + .set({ cursor: next, updatedAt: now }) + .where(and(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), eq(workosSync.cursor, prev))) + .returning({ id: workosSync.id }); + return written.length > 0; +}; + // Claim the organization for a scan listed at `listedAt`: move its // `backfilled_at` forward to `listedAt` if the recorded mark is older (or // missing). Run inside a transaction this also LOCKS the organization row @@ -875,31 +1094,21 @@ export const makeWorkOsMirrorStore = (db: DrizzleDb): WorkOsMirrorShape => { return rows[0]?.cursor ?? null; }), - setCursor: (prev, next) => - run("setCursor", async () => { - const now = new Date(); - if (prev === null) { - // First advance: mint the row, or claim an existing row that still - // has no cursor. A row that already carries one belongs to another - // run and is left alone. - const written = await db - .insert(workosSync) - .values({ id: WORKOS_EVENTS_STREAM_ID, cursor: next, updatedAt: now }) - .onConflictDoUpdate({ - target: workosSync.id, - set: { cursor: next, updatedAt: now }, - setWhere: isNull(workosSync.cursor), - }) - .returning({ id: workosSync.id }); - return written.length > 0; - } - const written = await db - .update(workosSync) - .set({ cursor: next, updatedAt: now }) - .where(and(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), eq(workosSync.cursor, prev))) - .returning({ id: workosSync.id }); - return written.length > 0; - }), + applyPage: (prev, next, pageWrites) => + run("applyPage", () => + db.transaction(async (tx) => { + // The CAS comes FIRST so the lock is held for every write below; + // a run that lost the stream commits an empty transaction. + const owned = await advanceCursor(tx, prev, next); + if (!owned) return Option.none(); + const txWrites = makeWrites(tx); + const outcomes: WorkOsMirrorWriteOutcome[] = []; + for (const write of pageWrites) { + outcomes.push(await applyWrite(txWrites, write)); + } + return Option.some(outcomes); + }), + ), applyOrganizationScan: (scan) => run("applyOrganizationScan", () => @@ -987,6 +1196,30 @@ export const makeWorkOsMirrorStore = (db: DrizzleDb): WorkOsMirrorShape => { return recorded.length > 0; }), + drainedAt: () => + run("drainedAt", async () => { + const rows = await db + .select({ drainedAt: workosSync.drainedAt }) + .from(workosSync) + .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID)); + return rows[0]?.drainedAt ?? null; + }), + + markDrained: (at) => + run("markDrained", async () => { + const moved = await db + .update(workosSync) + .set({ drainedAt: at }) + .where( + and( + eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), + or(isNull(workosSync.drainedAt), lt(workosSync.drainedAt, at)), + ), + ) + .returning({ id: workosSync.id }); + return moved.length > 0; + }), + organizationBackfilledAt: (organizationId) => run("organizationBackfilledAt", async () => { const rows = await db diff --git a/apps/cloud/src/auth/workos-mirror.node.test.ts b/apps/cloud/src/auth/workos-mirror.node.test.ts index 25e493385..f70c20816 100644 --- a/apps/cloud/src/auth/workos-mirror.node.test.ts +++ b/apps/cloud/src/auth/workos-mirror.node.test.ts @@ -26,7 +26,8 @@ // - a delete with no WorkOS instant keeps the row's own WorkOS stamp, so // a replacement membership WorkOS created meanwhile is not refused, // while the removed membership's own payload still is -// - the cursor advances only by compare-and-set (one owner per stream) +// - the cursor advances only by compare-and-set (one owner per stream), +// and a page that loses the CAS writes nothing // - a backfill scan is applied only if its listing is newer than the one // already applied to the organization (one owner per listing instant), // so an older listing cannot insert a membership the newer one lacked; @@ -52,7 +53,12 @@ import { DbService, makeDbLayer } from "../db/db"; import { accounts, organizations } from "../db/schema"; import { cloudMemberDirectoryLayer } from "./member-directory"; import { UserStoreService } from "./context"; -import { WorkOsMirror, type WorkOsMirrorMembership, type WorkOsMirrorUser } from "./workos-mirror"; +import { + WorkOsMirror, + WorkOsMirrorWrite, + type WorkOsMirrorMembership, + type WorkOsMirrorUser, +} from "./workos-mirror"; import { makeWorkOsMirrorStore } from "./workos-mirror-store"; const DbLive = DbService.Live; @@ -776,31 +782,67 @@ describe("WorkOsMirror upserts", () => { }); describe("WorkOsMirror cursor", () => { - it("advances only by compare-and-set", async () => { + it("advances only by compare-and-set, and a page that loses the CAS writes nothing", async () => { const result = await run( Effect.gen(function* () { const mirror = yield* WorkOsMirror; + const directory = yield* MemberDirectory; + const org = yield* freshOrg(); + const id = `user_${crypto.randomUUID()}`; // The cursor is instance-wide; read whatever a previous test left so // this test's expectations are relative, not absolute. const before = yield* mirror.getCursor(); - const first = yield* mirror.setCursor(before, "event_1"); - const wrongPrev = yield* mirror.setCursor(before === null ? "event_0" : null, "event_x"); + const first = yield* mirror.applyPage(before, "event_1", []); + const wrongPrev = yield* mirror.applyPage(before === null ? "event_0" : null, "event_x", [ + WorkOsMirrorWrite.UpsertUser({ user: user(id) }), + WorkOsMirrorWrite.UpsertMembership({ + membership: membership(org, id), + }), + ]); const afterWrong = yield* mirror.getCursor(); - const right = yield* mirror.setCursor("event_1", "event_2"); + const notWritten = yield* directory.membership(id, org); + const right = yield* mirror.applyPage("event_1", "event_2", [ + WorkOsMirrorWrite.UpsertUser({ user: user(id) }), + WorkOsMirrorWrite.UpsertMembership({ + membership: membership(org, id), + }), + // A rename of an org the mirror has never seen: nothing to write. + WorkOsMirrorWrite.RenameOrganization({ + organizationId: "org_nobody", + name: "Nobody", + updatedAt: T1, + }), + ]); const after = yield* mirror.getCursor(); - return { first, wrongPrev, afterWrong, right, after }; + const written = yield* directory.membership(id, org); + return { + id, + org, + first, + wrongPrev, + afterWrong, + notWritten, + right, + after, + written, + }; }), ); - expect(result.first).toBe(true); - expect(result.wrongPrev, "a run holding a stale prev cannot move the cursor").toBe(false); + expect(Option.isSome(result.first)).toBe(true); + expect( + Option.isNone(result.wrongPrev), + "a run holding a stale prev cannot move the cursor", + ).toBe(true); expect(result.afterWrong).toBe("event_1"); - expect(result.right).toBe(true); + expect(result.notWritten, "and none of its page's writes land").toBeNull(); + expect(result.right).toEqual(Option.some(["applied", "applied", "absent"])); expect(result.after).toBe("event_2"); + expect(result.written?.membershipId).toBe(`om_${result.id}_${result.org}`); }); }); describe("WorkOsMirror backfill sync state", () => { - it("records the replay boundary and the backfill completion once each, without touching the cursor", async () => { + it("records the replay boundary and the backfill completion once each, and the drained mark forward only, without touching the cursor", async () => { const result = await run( Effect.gen(function* () { const mirror = yield* WorkOsMirror; @@ -808,6 +850,8 @@ describe("WorkOsMirror backfill sync state", () => { // empty test database, other tests may have written it): start from // no row, as a database that has never been backfilled has. yield* clearEventsRow; + // No events row yet: nothing has been drained, and nothing is minted. + const drainedWithoutRow = yield* mirror.markDrained(T1); const first = yield* mirror.setReplayBoundary(T2); const boundary = yield* mirror.replayBoundary(); const cursorAfterBoundary = yield* mirror.getCursor(); @@ -816,7 +860,7 @@ describe("WorkOsMirror backfill sync state", () => { const afterAgain = yield* mirror.replayBoundary(); // Nor once the stream is being followed. const cursorBefore = yield* mirror.getCursor(); - yield* mirror.setCursor(cursorBefore, "event_boundary"); + yield* mirror.applyPage(cursorBefore, "event_boundary", []); const afterCursor = yield* mirror.setReplayBoundary(T1); const boundaryWithCursor = yield* mirror.replayBoundary(); const cursor = yield* mirror.getCursor(); @@ -828,7 +872,19 @@ describe("WorkOsMirror backfill sync state", () => { const completedAt = yield* mirror.backfillCompletedAt(); const boundaryAfterCompletion = yield* mirror.replayBoundary(); const cursorAfterCompletion = yield* mirror.getCursor(); + // The drained mark moves forward only, on the row the stream owns. + const notDrained = yield* mirror.drainedAt(); + const drainedFirst = yield* mirror.markDrained(T3); + const drainedBackwards = yield* mirror.markDrained(T2); + const drainedForward = yield* mirror.markDrained(T4); + const drainedAt = yield* mirror.drainedAt(); return { + drainedWithoutRow, + notDrained, + drainedFirst, + drainedBackwards, + drainedForward, + drainedAt, first, boundary, cursorAfterBoundary, @@ -860,6 +916,14 @@ describe("WorkOsMirror backfill sync state", () => { expect(result.completedAt, "the first stands").toEqual(T3); expect(result.boundaryAfterCompletion, "the boundary is untouched").toEqual(T2); expect(result.cursorAfterCompletion, "and so is the cursor").toBe("event_boundary"); + expect(result.drainedWithoutRow, "no row, nothing drained: nothing written").toBe(false); + expect(result.notDrained, "no drain recorded until a run drains").toBeNull(); + expect(result.drainedFirst).toBe(true); + expect(result.drainedBackwards, "an earlier run finishing later cannot move it back").toBe( + false, + ); + expect(result.drainedForward).toBe(true); + expect(result.drainedAt).toEqual(T4); }); it("refuses a membership payload stamped before the organization's last scan, and accepts one stamped at or after it", async () => { diff --git a/apps/cloud/src/auth/workos-mirror.ts b/apps/cloud/src/auth/workos-mirror.ts index 6898facdb..70ea8bf36 100644 --- a/apps/cloud/src/auth/workos-mirror.ts +++ b/apps/cloud/src/auth/workos-mirror.ts @@ -23,6 +23,7 @@ import { makeWorkOsMirrorStore, type WorkOsMirrorShape } from "./workos-mirror-s export { WorkOsMirrorError } from "./errors"; export { + WorkOsMirrorWrite, mirrorMembershipFromWorkOs, mirrorUserFromWorkOs, type WorkOsMembershipPayload, @@ -30,6 +31,7 @@ export { type WorkOsMirrorMembershipRef, type WorkOsMirrorShape, type WorkOsMirrorUser, + type WorkOsMirrorWriteOutcome, type WorkOsOrganizationScan, type WorkOsOrganizationScanWrites, type WorkOsScannedMember, diff --git a/apps/cloud/src/auth/workos-webhook.ts b/apps/cloud/src/auth/workos-webhook.ts new file mode 100644 index 000000000..5309cdc58 --- /dev/null +++ b/apps/cloud/src/auth/workos-webhook.ts @@ -0,0 +1,94 @@ +// --------------------------------------------------------------------------- +// `POST /api/webhooks/workos` — the WorkOS webhook endpoint, which only +// POKES the reconciler. It verifies the delivery's signature and, when it +// is genuine, starts one `syncWorkOsEvents` pass past the response. It +// never applies the webhook's own payload: webhooks are unordered and +// at-least-once, while the Events API the reconciler reads is ordered and +// replayable from the persisted cursor. The webhook's only job is to turn +// "within a minute" (the cron) into "within seconds" for dashboard-side +// changes such as a revoked membership. +// +// Unauthenticated by design (WorkOS holds no session); the signature IS the +// authentication. Nothing about the payload is reflected in the response. +// --------------------------------------------------------------------------- + +import { Effect, Option, Schema } from "effect"; +import { Headers, HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; + +import { WorkOSClient } from "./workos"; + +export const WORKOS_WEBHOOK_PATH = "/api/webhooks/workos"; + +const SIGNATURE_HEADER = "workos-signature"; + +// The SDK verifies the signature over `JSON.stringify(payload)`, so the body +// must be a JSON object; an array or scalar can never be a WorkOS delivery. +const WebhookPayload = Schema.Record(Schema.String, Schema.Unknown); +const decodeWebhookPayload = Schema.decodeUnknownOption(WebhookPayload); + +export interface WorkOsWebhookDeps { + /** + * The endpoint's signing secret (`WORKOS_WEBHOOK_SECRET`). `undefined` + * when the deployment has not configured one: every delivery is then + * refused with 503, never accepted unverified. + */ + readonly secret: string | undefined; + /** + * Hand the reconciler pass to the platform so it outlives the response + * (`waitUntil` from `cloudflare:workers`). The promise never rejects: the + * runner reports its own failures. + */ + readonly detach: (work: Promise) => void; + /** One reconciler pass over fresh services (`runWorkOsEventsSync`). */ + readonly sync: () => Promise; +} + +/** + * The webhook route. 200 for a verified delivery (a sync pass has been + * detached), 400 for a missing or invalid signature or a body that is not a + * JSON object, 503 when no signing secret is configured. + */ +export const makeWorkOsWebhookRoute = (deps: WorkOsWebhookDeps) => + HttpRouter.add( + "POST", + WORKOS_WEBHOOK_PATH, + Effect.gen(function* () { + if (deps.secret === undefined) { + yield* Effect.logError( + "workos_webhook: WORKOS_WEBHOOK_SECRET is not set; refusing the delivery", + ); + return HttpServerResponse.empty({ status: 503 }); + } + const secret = deps.secret; + const request = yield* HttpServerRequest.HttpServerRequest; + const sigHeader = Headers.get(request.headers, SIGNATURE_HEADER); + if (Option.isNone(sigHeader)) { + return HttpServerResponse.empty({ status: 400 }); + } + const body = yield* request.json.pipe(Effect.option); + const payload = Option.flatMap(body, decodeWebhookPayload); + if (Option.isNone(payload)) { + return HttpServerResponse.empty({ status: 400 }); + } + + const workos = yield* WorkOSClient; + const verified = yield* workos + .constructWebhookEvent({ + payload: payload.value, + sigHeader: sigHeader.value, + secret, + }) + .pipe(Effect.option); + if (Option.isNone(verified)) { + yield* Effect.logWarning("workos_webhook: signature rejected"); + return HttpServerResponse.empty({ status: 400 }); + } + + yield* Effect.logInfo("workos_webhook: verified delivery; poking the reconciler", { + event: verified.value.event, + eventId: verified.value.id, + }); + deps.detach(deps.sync()); + return HttpServerResponse.empty({ status: 200 }); + }), + ); diff --git a/apps/cloud/src/auth/workos.ts b/apps/cloud/src/auth/workos.ts index 821ea9027..dcaaa723d 100644 --- a/apps/cloud/src/auth/workos.ts +++ b/apps/cloud/src/auth/workos.ts @@ -4,7 +4,12 @@ import { env } from "cloudflare:workers"; import { Context, Data, Effect, Layer, Option, Predicate, Schema } from "effect"; -import { GeneratePortalLinkIntent, WorkOS } from "@workos-inc/node/worker"; +import { + GeneratePortalLinkIntent, + WorkOS, + type Event as WorkOSEvent, + type EventName as WorkOSEventName, +} from "@workos-inc/node/worker"; import { defaults as ironDefaults, unseal as unsealIron } from "iron-webcrypto"; import { decodeJwt, jwtVerify } from "jose"; import { workosAccessTokenOptions } from "./access-token-options"; @@ -16,6 +21,7 @@ import { tryPromiseService, withServiceLogging, workosErrorFromFailure, + type WorkOSError, } from "./errors"; const COOKIE_NAME = "wos-session"; @@ -48,6 +54,20 @@ type WorkOSAutoPaginatable = { readonly autoPagination: () => Promise; }; +/** + * One read of the WorkOS Events API stream. `events` names the types to + * return; `after` resumes from an event id (exclusive), `rangeStart` (ISO) + * bounds a first read that has no cursor yet. Mirrors the SDK's + * `ListEventOptions` with readonly inputs. + */ +export type WorkOSListEventsOptions = { + readonly events: readonly WorkOSEventName[]; + readonly after?: string; + readonly rangeStart?: string; + readonly limit?: number; + readonly order?: "asc" | "desc"; +}; + export type WorkOSCollectedList = { readonly object: "list"; readonly data: Resource[]; @@ -753,6 +773,48 @@ const make = Effect.gen(function* () { wos.organizations.listOrganizationRoles({ organizationId }), ), + /** + * One page of the Events API stream, oldest first when `order` is `asc`. + * The reconciler (`workos-events-sync.ts`) is the only consumer: it pages + * by `after` = the last event id it applied, so the stream is replayable + * from the persisted cursor. Returns the SDK page as-is (`data` + + * `listMetadata.after`); paging is the caller's loop, not + * `collectWorkOSList`, because each page is committed before the next is + * read. + */ + listEvents: (options: WorkOSListEventsOptions) => + use("events.listEvents", (wos) => + wos.events.listEvents({ + events: [...options.events], + ...(options.after === undefined ? {} : { after: options.after }), + ...(options.rangeStart === undefined ? {} : { rangeStart: options.rangeStart }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.order === undefined ? {} : { order: options.order }), + }), + ), + + /** + * Verify a webhook delivery against `secret` (the endpoint's signing + * secret from the WorkOS dashboard) and decode its event. A local HMAC + * check, no network: it fails with a status-less `WorkOSError` when the + * `WorkOS-Signature` header is missing its parts, older than the SDK's + * tolerance, or does not match `payload`. The decoded event is returned + * for the caller to inspect; the webhook route deliberately does NOT + * apply it (the Events API is the only source the mirror replays from). + */ + constructWebhookEvent: (params: { + readonly payload: Record; + readonly sigHeader: string; + readonly secret: string; + }): Effect.Effect => + use("webhooks.constructEvent", (wos) => + wos.webhooks.constructEvent({ + payload: params.payload, + sigHeader: params.sigHeader, + secret: params.secret, + }), + ), + /** Get an organization (includes domains). */ getOrganization: (organizationId: string) => use("organizations.getOrganization", (wos) => diff --git a/apps/cloud/src/db/schema.ts b/apps/cloud/src/db/schema.ts index 9bbd73290..89e5c3cdb 100644 --- a/apps/cloud/src/db/schema.ts +++ b/apps/cloud/src/db/schema.ts @@ -83,13 +83,14 @@ export const organizations = pgTable( backfilledAt: timestamp("backfilled_at", { withTimezone: true }), /** * When this organization was deleted, or null while it is live. Set by - * cloud's own deletion flow and by the `organization.deleted` event, and - * KEPT by the local purge (`db/org-deletion.ts`), which removes the - * organization's memberships and tenant data but leaves this row as a - * tombstone: a feeder that fetched a membership before the deletion and - * writes it after (a login that stalled across the purge) finds the - * tombstone and does not re-mint the organization live. A marked - * organization is never renamed and authorizes nobody. + * cloud's own deletion flow and by the `organization.deleted` event — + * which MINTS the row as a tombstone when the mirror has never seen the + * organization — and KEPT by the local purge (`db/org-deletion.ts`), + * which removes the organization's memberships and tenant data but + * leaves this row as a tombstone: a feeder that fetched a membership + * before the deletion and writes it after (a login that stalled across + * the deletion) finds the tombstone and does not mint the organization + * live. A marked organization is never renamed and authorizes nobody. */ deletedAt: timestamp("deleted_at", { withTimezone: true }), /** @@ -201,16 +202,16 @@ export const membershipTombstones = pgTable( * stops. * * `range_start` on the `"events"` row is the REPLAY BOUNDARY: the instant the - * FIRST completed one-off backfill (`scripts/backfill-workos-mirror.ts`) began - * reading WorkOS. Everything before it is covered by that backfill; the - * reconciler's first run (no cursor yet) reads the events stream from here, - * so a revocation between the backfill and the first run is never skipped. - * Written once: a backfill that fails part-way records nothing, and a later - * completed one keeps it, because the backfill does not refresh everything - * the events stream carries (organization renames, deleted users' - * profiles) — those between two runs are replayed from the first boundary. - * Without a cursor or a boundary the reconciler does not guess; it waits for - * the backfill. + * FIRST one-off backfill run (`scripts/backfill-workos-mirror.ts`) began + * reading WorkOS, recorded BEFORE its first listing. Everything before it is + * covered by that backfill; the reconciler's first run (no cursor yet) reads + * the events stream from here, so a revocation between the backfill and the + * first run is never skipped. Written once: a run that fails part-way leaves + * it standing and its retry keeps it, and a later run keeps it too, because + * the backfill does not refresh everything the events stream carries + * (organization renames, deleted users' profiles) — those after the first + * boundary are replayed from it. Without a cursor or a boundary the + * reconciler does not guess; it waits for the backfill. * * `backfill_completed_at` is when a backfill run first wrote EVERY live * organization (`scripts/backfill-workos-mirror.ts` completing, or refusing @@ -223,13 +224,23 @@ export const membershipTombstones = pgTable( * completeness for the seat gates is tracked separately * (`organizations.backfilled_at`). * + * `drained_at` is when a reconciler run last read the events stream to its + * END (an empty page, or a page with nothing after it) — the second half of + * the readiness mark: a mirror whose reconciler has not caught up recently + * may still grant a member WorkOS already revoked, so the authorization + * path trusts the mirror only while this is within its lag budget. Moved + * forward by every draining run; never cleared. A run that stops at its + * page budget or yields to another run leaves it as it was. + * * Migration 0019 seeds the boundary and the completion mark on a database - * with no organizations, where there is nothing to backfill. + * with no organizations, where there is nothing to backfill; `drained_at` + * is left for the reconciler's first run to set (migration 0020). */ export const workosSync = pgTable("workos_sync", { id: text("id").primaryKey(), cursor: text("cursor"), rangeStart: timestamp("range_start", { withTimezone: true }), backfillCompletedAt: timestamp("backfill_completed_at", { withTimezone: true }), + drainedAt: timestamp("drained_at", { withTimezone: true }), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }); diff --git a/apps/cloud/src/env-augment.d.ts b/apps/cloud/src/env-augment.d.ts index 773ca4d46..715991f39 100644 --- a/apps/cloud/src/env-augment.d.ts +++ b/apps/cloud/src/env-augment.d.ts @@ -111,6 +111,14 @@ declare global { /** Optional WorkOS base-URL override (WorkOS emulator in tests/dev). */ WORKOS_API_URL?: string; + /** + * Signing secret of the WorkOS webhook endpoint that pokes the + * membership-mirror reconciler (`/api/webhooks/workos`). Set with + * `wrangler secret put WORKOS_WEBHOOK_SECRET`; while unset the route + * refuses every delivery (503) and the every-minute cron alone keeps + * the mirror current. + */ + WORKOS_WEBHOOK_SECRET?: string; // MCP EXECUTOR_MCP_DEBUG?: string; diff --git a/apps/cloud/src/extensions/routes.ts b/apps/cloud/src/extensions/routes.ts index e362b62d5..bd4b1d4c1 100644 --- a/apps/cloud/src/extensions/routes.ts +++ b/apps/cloud/src/extensions/routes.ts @@ -9,6 +9,8 @@ // - Swagger UI + the OpenAPI JSON for the full cloud spec. // - the Autumn billing proxy (`/api/billing/*`) — billing-as-extension (the // `extensions.routes` SEAM, but served under `/api` like everything else). +// - the WorkOS webhook (`/api/webhooks/workos`) — signature-verified poke of +// the membership-mirror reconciler. // - the global request-failure logging middleware. // // They all serve UNDER the `/api` prefix (the same namespace the protected + @@ -19,6 +21,7 @@ // so the postgres.js socket lives in the request fiber's scope). // --------------------------------------------------------------------------- +import { env, waitUntil } from "cloudflare:workers"; import { Effect, Layer } from "effect"; import { HttpRouter, HttpServerResponse } from "effect/unstable/http"; import { HttpApiBuilder } from "effect/unstable/httpapi"; @@ -36,6 +39,8 @@ import { } from "../auth/handlers"; import { CloudAuthApi, CloudAuthPublicApi } from "../auth/api"; import { SessionAuthLive } from "../auth/middleware-live"; +import { runWorkOsEventsSync } from "../auth/workos-events-runner"; +import { makeWorkOsWebhookRoute } from "../auth/workos-webhook"; import { makeCloudAdminUsersRoutes } from "../admin/admin-users-api"; import { OrgApi, OrgHttpApi } from "../org/api"; import { orgAuthMiddleware } from "../org/auth-middleware"; @@ -113,12 +118,23 @@ export const makeCloudExtensionRoutes = ( // org key (or an admin session) and builds a subject-less platform view. const AdminUsersRoutes = makeCloudAdminUsersRoutes(rsLive, { router: apiPrefixedRouter }); + // The WorkOS webhook needs no per-request DB layer: it verifies the + // signature with the boot `WorkOSClient` and detaches a reconciler pass + // that builds its own fresh services (the route's request scope is gone by + // the time the pass runs). `waitUntil` binds to the in-flight invocation. + const WebhookRoutes = makeWorkOsWebhookRoute({ + secret: env.WORKOS_WEBHOOK_SECRET, + detach: waitUntil, + sync: runWorkOsEventsSync, + }); + return [ SessionRoutes, OrgRoutes, AdminUsersRoutes, DocsRoutes, BillingRoutes, + WebhookRoutes, ApiErrorLoggingLive, ] as const; }; diff --git a/apps/cloud/src/server.ts b/apps/cloud/src/server.ts index 4f5bf7628..fc9c146f3 100644 --- a/apps/cloud/src/server.ts +++ b/apps/cloud/src/server.ts @@ -13,6 +13,7 @@ import handler from "@tanstack/react-start/server-entry"; import { isAppOwnedPath, servedByAppPlane } from "./app-paths"; import { marketingProxyRequest } from "./edge/marketing"; import { passthroughResponse } from "./edge/passthrough"; +import { runWorkOsEventsSync } from "./auth/workos-events-runner"; import { makeCloudMcpAgentHandler } from "./mcp/agent-handler"; import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount"; import { parseTraceparent } from "./mcp/traceparent"; @@ -433,6 +434,20 @@ const cloudflareHandler: ExportedHandler = { }, ); }, + + // Cron: the membership-mirror reconciler (wrangler.jsonc `triggers.crons`, + // every minute). One pass over the WorkOS Events API from the persisted + // cursor, on fresh request-scoped services. `Sentry.withSentry` instruments + // `scheduled` alongside `fetch` (`instrumentExportedHandlerScheduled`), so + // a failing pass reports like a failing request. The tracer is installed + // here as on the fetch path — a scheduled invocation may be the isolate's + // first — and flushed past the pass so the run's spans export before the + // isolate goes idle. + scheduled: async (_controller, _env, ctx) => { + installTracerProvider(); + await runWorkOsEventsSync(); + ctx.waitUntil(flushTracerProvider()); + }, }; export default Sentry.withSentry(cloudSentryOptions, cloudflareHandler); diff --git a/apps/cloud/wrangler.jsonc b/apps/cloud/wrangler.jsonc index 3578de6d2..6d9206458 100644 --- a/apps/cloud/wrangler.jsonc +++ b/apps/cloud/wrangler.jsonc @@ -18,6 +18,13 @@ "limits": { "cpu_ms": 30000, }, + // Every minute: replay the WorkOS Events API into the membership mirror + // (`scheduled` in src/server.ts → auth/workos-events-sync.ts). Changes made + // in the WorkOS dashboard reach the mirror within this interval; the + // signed webhook at /api/webhooks/workos shortens it to seconds. + "triggers": { + "crons": ["* * * * *"], + }, "observability": { "enabled": true, },