diff --git a/alembic/versions/0007_worker_execution.py b/alembic/versions/0007_worker_execution.py new file mode 100644 index 0000000..b9bed73 --- /dev/null +++ b/alembic/versions/0007_worker_execution.py @@ -0,0 +1,43 @@ +"""Add concurrency-safe active-job deduplication for the PostgreSQL worker. + +Revision ID: 0007_worker_execution +Revises: 0006_governance_retention +""" + +from alembic import op + + +revision = "0007_worker_execution" +down_revision = "0006_governance_retention" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE job_runs ADD COLUMN active_dedupe_key TEXT") + op.execute( + """ + ALTER TABLE job_runs + ADD CONSTRAINT ck_job_runs_active_dedupe_key + CHECK ( + active_dedupe_key IS NULL + OR (btrim(active_dedupe_key) <> '' AND length(active_dedupe_key) <= 200) + ) + """ + ) + op.execute( + """ + CREATE UNIQUE INDEX uq_job_runs_active_dedupe + ON job_runs (active_dedupe_key) + WHERE active_dedupe_key IS NOT NULL + AND status IN ('queued', 'running') + """ + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS uq_job_runs_active_dedupe") + op.execute( + "ALTER TABLE job_runs DROP CONSTRAINT IF EXISTS ck_job_runs_active_dedupe_key" + ) + op.execute("ALTER TABLE job_runs DROP COLUMN IF EXISTS active_dedupe_key") diff --git a/config/observability.json b/config/observability.json index 85118ae..f3db7ea 100644 --- a/config/observability.json +++ b/config/observability.json @@ -1,7 +1,7 @@ { "configuration_version": "1", "service": "foundation-intelligence-api", - "expected_schema_version": "0006_governance_retention", + "expected_schema_version": "0007_worker_execution", "metrics": [ {"name": "readiness_success", "type": "gauge", "unit": "None", "dimensions": ["service", "environment"]}, {"name": "api_request_duration_ms", "type": "histogram", "unit": "Milliseconds", "dimensions": ["service", "environment", "operation", "status"]}, diff --git a/docker-compose.yml b/docker-compose.yml index f1e9445..83c5a14 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,8 @@ x-backend-common: &backend-common DATABASE_NAME: foundation_intelligence DATABASE_USER: foundation_app DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + DATABASE_WRITE_USER: foundation_app + DATABASE_WRITE_PASSWORD_FILE: /run/secrets/postgres_password DATABASE_CONNECT_TIMEOUT_SECONDS: "5" DATABASE_STATEMENT_TIMEOUT_MS: "30000" secrets: @@ -114,15 +116,24 @@ services: worker: <<: *backend-common - profiles: ["operations"] depends_on: postgres: condition: service_healthy - migration: - condition: service_completed_successfully entrypoint: ["python", "-m"] - command: ["pipelines.run_pipeline", "--source", "consolidate"] - restart: "no" + command: ["pipelines.worker"] + environment: + <<: *backend-environment + DATABASE_PIPELINE_USER: foundation_app + DATABASE_PIPELINE_PASSWORD_FILE: /run/secrets/postgres_password + CODE_REVISION: 0000000000000000000000000000000000000000 + PIPELINE_SOURCE_SCHEMA_VERSION: "7" + PIPELINE_BASELINE_PATH: /runtime-baseline/charities.db + PIPELINE_SNAPSHOT_PATH: /pipeline-snapshot/charities.db + PIPELINE_FALLBACK_SHA256: 8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651 + volumes: + - ./src/data/charities.db:/runtime-baseline/charities.db:ro + - pipeline_snapshot:/pipeline-snapshot + restart: unless-stopped secrets: postgres_password: @@ -130,3 +141,4 @@ secrets: volumes: postgres_data: + pipeline_snapshot: diff --git a/docker/frontend-nginx.conf b/docker/frontend-nginx.conf index d53e2bd..591f8ce 100644 --- a/docker/frontend-nginx.conf +++ b/docker/frontend-nginx.conf @@ -39,6 +39,6 @@ server { try_files $uri $uri/ /index.html; add_header X-Content-Type-Options nosniff always; add_header Referrer-Policy strict-origin-when-cross-origin always; - add_header Content-Security-Policy "default-src 'self'; img-src 'self' data: https:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self'; font-src 'self' data:; object-src 'none'; base-uri 'self'; frame-ancestors 'none'" always; + add_header Content-Security-Policy "default-src 'self'; img-src 'self' data: https:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self' https://*.auth.eu-west-1.amazoncognito.com; font-src 'self' data:; object-src 'none'; base-uri 'self'; frame-ancestors 'none'" always; } } diff --git a/docker/frontend-nginx.ecs.conf b/docker/frontend-nginx.ecs.conf index 5d2067d..ce96bd7 100644 --- a/docker/frontend-nginx.ecs.conf +++ b/docker/frontend-nginx.ecs.conf @@ -39,6 +39,6 @@ server { try_files $uri $uri/ /index.html; add_header X-Content-Type-Options nosniff always; add_header Referrer-Policy strict-origin-when-cross-origin always; - add_header Content-Security-Policy "default-src 'self'; img-src 'self' data: https:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self'; font-src 'self' data:; object-src 'none'; base-uri 'self'; frame-ancestors 'none'" always; + add_header Content-Security-Policy "default-src 'self'; img-src 'self' data: https:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self' https://*.auth.eu-west-1.amazoncognito.com; font-src 'self' data:; object-src 'none'; base-uri 'self'; frame-ancestors 'none'" always; } } diff --git a/docs/remediation/aws-postgres-route-inventory.md b/docs/remediation/aws-postgres-route-inventory.md index af4ad29..6045627 100644 --- a/docs/remediation/aws-postgres-route-inventory.md +++ b/docs/remediation/aws-postgres-route-inventory.md @@ -68,10 +68,13 @@ There is no internal-service route in the current application. Queue/task callba ## Phase boundary Production/staging application data, job state, request idempotency, link -overrides, profile caches and audit records use PostgreSQL. Manual refresh +overrides and profile caches use PostgreSQL through separate reader/writer +pools. Security audit events use structured application logs so a GET never +causes PostgreSQL DML. Manual refresh routes only enqueue durable jobs and return a job ID. Phase-8 workers claim through PostgreSQL leases and the transactional outbox supplies the SQS delivery contract; neither path uses a production local lock or API -subprocess. Production audit events are append-only PostgreSQL rows; local -unit tests may replace the sink with deterministic memory state. +subprocess. The exact mutation/session/table contract is documented in +`database-access-architecture.md`; local unit tests may replace durable stores +with deterministic memory state. Edge/distributed rate limiting remains a Terraform and deployment concern. diff --git a/docs/remediation/aws-public-demo-runbook.md b/docs/remediation/aws-public-demo-runbook.md index afb60e2..58cd9e3 100644 --- a/docs/remediation/aws-public-demo-runbook.md +++ b/docs/remediation/aws-public-demo-runbook.md @@ -1,212 +1,229 @@ -# Temporary AWS public read-only demo - -Status: design and read-only preflight snapshot prepared on 2026-08-06. This is -a temporary non-production demo design. Its HTTP listener is not -production-ready and must never transport login cookies, tokens or credentials. -This document describes controlled phases; it is not evidence that a particular -phase has or has not been executed. - -## Confirmed read-only preflight - -- AWS profile: `netlight` -- Account: `208337080387` (exact match) -- Region/configured region: `eu-west-1` (exact match) -- Default VPC: `vpc-0348ad96c2afcf8dd`, `172.31.0.0/16` -- Internet gateway: `igw-08ecd35c2bbf38c46` -- Public ALB/ECS subnets: - - `subnet-04d4e3e5af84a7ea4`, `eu-west-1a`, `172.31.16.0/20` - - `subnet-076f66fcdd86cd28f`, `eu-west-1b`, `172.31.32.0/20` - - spare confirmed public subnet: `subnet-0f5b48470ee59b581`, - `eu-west-1c`, `172.31.0.0/20` -- All three default subnets map public IPs and inherit the active - `0.0.0.0/0 -> igw-08ecd35c2bbf38c46` route. -- No isolated RDS subnets exist. The template therefore creates - `172.31.240.0/28` in `eu-west-1a` and `172.31.240.16/28` in `eu-west-1b`, - associated only with a new route table whose sole route is VPC-local. -- Existing ECR repositories: - - `foundation-intelligence-platform/backend` - - `foundation-intelligence-platform/frontend` -- Both repositories currently contain zero images. The public AWS CLI - `2.27.41` manifest was checked and contains both linux/amd64 and linux/arm64. -- PostgreSQL `16.14`, gp3, `db.t4g.medium` and `db.t4g.small` are orderable in - `eu-west-1a`, `eu-west-1b` and `eu-west-1c`. -- No matching CloudFormation stacks, ECS clusters, RDS instances, ALBs, - Secrets Manager secrets, CloudWatch log groups, S3 buckets, IAM roles, - schedules or security groups were found. -- `iam:SimulatePrincipalPolicy` was denied for the current user. Per the - requested contract the IAM result is `UNKNOWN`; no create call was used as a - permission test. - -The existing ECR repositories are mutable-tag repositories with AES256 -encryption and scan-on-push. Deployment commands must use unique Git-SHA tags -and resolve the resulting image digests before the stack is updated; `latest` -is prohibited by the template input contract. - -## Resource and sizing contract - -The template is `infra/cloudformation/demo.yaml`. - -| Component | Exact demo configuration | -|---|---| -| Application task | Fargate Linux/ARM64, 512 CPU units, 2,048 MiB, default 20 GiB ephemeral storage, desired count 0 initially then 1 | -| Frontend | Same task, unprivileged port 8080 behind the ALB listener on port 80; Nginx proxies `/api` to `127.0.0.1:8000` | -| Backend | Same task, port 8000 not exposed by any security-group ingress rule | -| Migration task | Fargate Linux/ARM64, 1,024 CPU units, 4,096 MiB, default 20 GiB ephemeral storage, one-off only | -| RDS import | PostgreSQL 16.14, `db.t4g.medium`, Single-AZ, 30 GiB gp3, autoscaling maximum 100 GiB | -| RDS steady state | Change the same stack parameter to `db.t4g.small` after import and loaded smoke tests | -| RDS safety | private/isolated subnets, public access false, encrypted storage, forced TLS, one-day backup retention, snapshot on delete/replace | -| ALB | internet-facing, two public subnets, HTTP port 80, IP targets, `/health/live` | -| S3 | private, bucket-owner enforced, AES256, full public-access block, TLS-only policy, import objects expire after seven days | -| Logs | separate frontend, backend and migration log groups; seven-day retention | - -There is no NAT gateway, autoscaling, Multi-AZ database, replica, Performance -Insights or VPC endpoint. The application task receives a public IP solely for -outbound AWS/API access. Its inbound port 8080 accepts traffic only from the ALB -security group. RDS port 5432 accepts traffic only from the shared ECS security -group. - -The demo uses asyncpg's `ssl=require` mode for every PostgreSQL connection. It -encrypts transport to RDS but does not verify the server identity. Production -hardening should use `verify-full` with the AWS RDS CA certificate after that -mode and certificate distribution are added to the runtime contract. - -The RDS-managed master secret is injected only into the one-off migration task. -CloudFormation generates a second application secret. The migration bootstrap -applies Alembic and runs the existing migration as the master, then -creates/rotates `foundation_app` without superuser, role-creation or DDL -capabilities. The application role receives only database connect, schema usage -and table `SELECT`; its transactions default to read-only. Public application -startup also skips configuration synchronization and database-backed audit or -idempotency writes. The runtime task never receives the master secret. - -## Source and migration gate - -The approved local source was independently rechecked: - -- path: `src/data/charities.db` -- size: `2,100,543,488` bytes -- SHA-256: `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651` -- source schema version: `7` -- SQLite quick check: `ok` -- PostgreSQL Alembic head: `0006_governance_retention` - -The migration task has three ordered containers: - -1. An AWS CLI init container downloads exactly `imports/charities.db` from the - stack's private S3 bucket into the shared 20 GiB task volume, creates the - evidence directory and hands the volume to the non-root migration UID. -2. `migration.aws_entrypoint` acquires a PostgreSQL advisory lock, applies - Alembic and invokes the existing `migration.sqlite_to_postgres.migrate` - workflow with the master secret, then configures the SELECT-only application - role and verifies the release gate through that role. SHA-256 and source - schema are checked before loading. The remote-RDS capacity mode counts only - the source plus a 4 GiB local safety margin; PostgreSQL data and WAL remain - on the 30–100 GiB RDS volume and are not incorrectly counted against Fargate. -3. Only after migration success, a final AWS CLI container uploads the generated - reconciliation evidence to `migration-evidence//`. - -The existing loader stages a versioned candidate, validates types and foreign -keys, builds analytics materializations, reconciles baseline counts and controls, -and activates in a transaction only if every reconciliation passes. On any -failure the candidate is failed/rejected and the previous active dataset remains -unchanged. The release gate must also report ready before the task succeeds. - -## Controlled phase sequence - -Values such as the reviewed Git SHA and created bucket/ARNs must be resolved at -execution time; they must not be guessed. - -1. Review, then locally commit only the listed repository changes. Do not push - to GitHub during Phase A. -2. Build `backend-runtime` and `frontend-runtime-ecs` for `linux/arm64`. -3. Authenticate Docker to the two existing private ECR repositories, tag each - image once with `demo-`, push, then read back and use immutable - image digests. No ECR repository creation is planned. -4. Create, inspect and stop at a CloudFormation change set using - `CAPABILITY_IAM`, the reviewed image digests, full Git SHA, - `ApplicationDesiredCount=0` and `DatabaseInstanceClass=db.t4g.medium`. - Do not execute the change set during Phase A. -5. In a separately approved later phase, execute the reviewed change set. Read - the generated `ImportBucketName`, then upload exactly - `src/data/charities.db` to `s3:///imports/charities.db`, with the - approved SHA-256 recorded as object metadata, then read back object metadata. -6. Run exactly one `MigrationTaskDefinition` task in the two confirmed public - subnets with `AssignPublicIp=ENABLED` and the stack ECS security group. Wait - for all containers to exit zero and inspect the migration logs/evidence. -7. Keep the web service at zero until release-gate, readiness and loaded API - smoke checks pass. Then execute a second reviewed stack update setting only - `ApplicationDesiredCount=1`. -8. After import and loaded browser/API tests pass, execute a third reviewed - stack update changing RDS from `db.t4g.medium` to `db.t4g.small`. Do not use - `ApplyImmediately`; inspect the planned maintenance behavior first. - -S3 upload, change-set execution, CloudFormation deployment, ECS `RunTask`, RDS -modification, GitHub push and resource deletion are outside Phase A. - -## Automatic refresh plan — intentionally not scheduled - -The repository currently has solid versioned migration, reconciliation, -transactional activation, durable PostgreSQL job/outbox/lease primitives and -source limits. It does not yet have a production executable that connects the -current filesystem-oriented scraper/consolidation commands to immutable S3 and -a PostgreSQL candidate dataset. All source configurations are also disabled and -governance-blocked with unresolved legal/licence status. Creating a task or -schedule that pretends otherwise would not be fail-closed. - -After source governance approval, implement and manually validate this separate -ARM64 Fargate refresh task; the web service must never run it in the background: - -`EventBridge Scheduler (disabled initially)` → `ECS Refresh Task` → acquire -PostgreSQL advisory lock → fetch bounded approved sources into immutable S3 keys -→ validate schema/checksum and deduplicate → build a new versioned candidate → -reconcile counts/quality/materializations → transactionally activate → release -lock. - -- Proposed task: 1,024 CPU units, 4,096 MiB, default 20 GiB ephemeral storage, - public subnet/public IP for bounded source egress, no inbound rule. -- Parallel runs: `pg_try_advisory_lock` plus the durable job lease/unique - idempotency key; a second run exits without scraping or changing data. -- Failure isolation: source and candidate identifiers are immutable/versioned; - active-dataset pointers change only in the final successful transaction. -- Retry: task-level maximum two retries with exponential backoff for transient - ECS/source failures; no retry for validation, governance or reconciliation - failures; CloudWatch records class/counts without raw payloads or credentials. -- Initial scheduler state: absent/disabled until a manual AWS refresh succeeds. -- Later weekly default: `cron(0 2 ? * MON *)` in UTC, maximum event age one hour, - maximum two delivery retries. Enable only in a separately reviewed stack - update after the manual gate. - -## Monthly cost estimate - -The estimate uses the AWS Price List API values returned for EU (Ireland) on -2026-08-06, 730 hours/month, on-demand pricing and a low-traffic assumption of -0.1 average ALB LCU. It excludes VAT, support, internet egress and unexpected -traffic. CloudWatch assumes 1 GiB/month of logs; ECR assumes roughly 2 GiB of -images. Public IPv4 assumes two ALB addresses plus one running application task. - -| Item | Initial import month | After RDS resize | -|---|---:|---:| -| RDS instance | $50.37 (`db.t4g.medium`) | $25.55 (`db.t4g.small`) | -| RDS 30 GiB gp3 | $3.81 | $3.81 | -| Fargate app 0.5 vCPU / 2 GiB | $17.02 | $17.02 | -| ALB hours | $18.40 | $18.40 | -| ALB 0.1 average LCU assumption | $0.58 | $0.58 | -| Three public IPv4 addresses | $10.95 | $10.95 | -| Two Secrets Manager secrets | $0.80 | $0.80 | -| CloudWatch, ECR and temporary S3 estimate | $0.78 | $0.78 | -| **Estimated total** | **about $103/month** | **about $78/month** | - -The one-off 1 vCPU/4 GiB migration task is approximately $0.04662 per task-hour -plus one public IPv4 at $0.005/hour; even a six-hour import adds less than $0.32. -At one full ALB LCU instead of 0.1, add approximately $5.26/month. Internet data -transfer and request-driven load remain variable and require a budget/alarm -before any broader demo distribution. - -## Hardening after the temporary demo - -Before any production claim or use of login credentials, add an approved domain, -ACM certificate and HTTPS listener; redirect HTTP to HTTPS; replace the public -read-only mode with OIDC; add edge abuse protection/budgets; complete privacy and -source-governance approval; test restore/rollback; and reassess Multi-AZ, backups, -deletion protection and private ECS networking. None of those production claims -apply to the temporary HTTP demo. +# CloudFront and Cognito customer-pilot runbook + +Status: repository-only implementation prepared and locally validated on +2026-08-12. No change set, deployment, Cognito user, image push, data operation +or other AWS write is evidenced by this document. + +## Confirmed baseline + +- AWS profile/account/region: `netlight`, `208337080387`, `eu-west-1`. +- Existing stack: `foundation-intelligence-demo-e06ab1ea-v2`, observed in + `UPDATE_COMPLETE` before this implementation. +- Existing path: internet -> public ALB HTTP 80 -> ECS frontend 8080 -> nginx -> + backend `127.0.0.1:8000` -> private RDS PostgreSQL. +- ECS port 8080 accepts only the ALB security group. Backend port 8000 has no + direct ingress. RDS port 5432 accepts only the ECS security group. +- The confirmed CloudFront origin-facing prefix list in `eu-west-1` was + `pl-4fa04526`, weight 55. This identifier is deliberately not hardcoded in the + template and must be resolved again immediately before Deployment B. +- The ALB security-group inbound quota was 60 weighted rules. The observed + current weight was 1, expected State B weight 55, and transition maximum 56. +- At preflight there were no CloudFront distributions, Cognito user pools or + Cognito domains in scope. + +## Architecture and accepted temporary residual risk + +The customer address is the generated `https://.cloudfront.net` +URL. There is no alias, custom domain, Route 53 record or ACM certificate in the +active pilot path. + +```text +Browser --HTTPS--> CloudFront --HTTP:80--> ALB --HTTP--> ECS nginx/backend + | + +--> RDS (TLS required) +``` + +**ACCEPTED TEMPORARY RESIDUAL RISK:** CloudFront-to-ALB and ALB-to-ECS are +HTTP-only. This is not end-to-end TLS. In particular, bearer tokens are not +protected by TLS on the CloudFront-to-ALB hop. The origin-verification header is +an access-control aid, not transport encryption and not a permanently +unreadable secret. + +The later hardening option is a reviewed custom domain plus ACM certificate, +HTTPS from CloudFront to the ALB, and origin-certificate validation. Earlier +repository work for direct ALB HTTPS informed this option, but +`CertificateArn` and `CustomDomainName` do not block the no-domain pilot. + +## Authorization contract + +The backend is the authoritative security boundary. Frontend visibility is UX +only. A validated Cognito access token must contain exactly one of the three +application groups; zero or multiple app groups fail closed with HTTP 403. + +| Capability | customer | operator | admin | +|---|:---:|:---:|:---:| +| Analytics, maps, directories, search, details and drill-downs | yes | yes | yes | +| Sanitized scraper status, sources, freshness and counts | yes | yes | yes | +| Scraper/pipeline run and operational actions | no | yes | yes | +| Relevant operational observability | no | yes | yes | +| Pipeline logs, governance administration and critical settings | no | no | yes | +| Cognito users and application roles | no | no | yes | + +User-management paths use Cognito `Username` as their stable opaque `{id}`. +Email is display/invitation data, never the administrative URL identifier. +Hard delete is intentionally absent. The backend blocks self-disable, +self-downgrade, last-active-admin disable and last-active-admin downgrade. + +## Cognito and browser login contract + +- User Pool sign-in identifier: email; self-registration is disabled. +- Password: at least 12 characters with lower/upper case, number and symbol. +- Recovery: verified email; no SMS dependency. +- MFA: `MfaConfiguration: 'ON'` and only + `EnabledMfas: [SOFTWARE_TOKEN_MFA]`. `EnabledMfas` is the CloudFormation + property that activates TOTP; `SoftwareTokenMfaConfiguration` belongs to the + Cognito service API and is not a valid `AWS::Cognito::UserPool` property. +- Managed Login v2 with Cognito-provided branding. +- Public app client, no client secret, authorization-code flow only, PKCE S256, + scopes `openid email profile`; implicit flow is disabled. +- Callback: `https://${CloudFrontDistribution.DomainName}/auth/callback`. +- Logout: `https://${CloudFrontDistribution.DomainName}/`. +- The frontend keeps tokens only in session storage, sends the access token as + the API bearer token, uses the ID token only for display email, never logs + tokens, validates OAuth state and removes callback parameters immediately. + +AWS documents that required MFA makes Managed Login prompt users to configure +an additional factor. With email invitation and the configured pool, the first +login sequence is invitation -> temporary password -> new password -> TOTP +enrolment -> authenticated redirect. This must still be manually confirmed in +Deployment A before origin lockdown. + +## CloudFront contract + +- Default CloudFront certificate, no aliases, HTTPS redirect, HTTP/2 and HTTP/3. +- Default behavior allows `GET`, `HEAD`, `OPTIONS`; caching is disabled and all + callback query strings are forwarded. +- `/api/*` allows all required methods. Both behaviors use AWS managed + `Managed-CachingDisabled` (`4135ea2d-6df8-44a3-9df3-4b5a84be39ad`), avoiding + invalid custom zero-TTL cache-policy combinations. +- The default behavior uses a minimal custom origin-request policy: all query + strings, no viewer cookies and no viewer headers. +- `/api/*` uses AWS managed `Managed-AllViewerExceptHostHeader` + (`b689b0a8-53d0-40ab-baf2-68738e2966ac`). It forwards `Authorization` and + other viewer request context while replacing the viewer `Host` with the ALB + origin host. API caching remains disabled. This managed policy forwards + cookies as a documented tradeoff even though application auth uses bearer + tokens rather than cookies. +- The ALB origin is obtained from `LoadBalancer.DNSName`, uses HTTP port 80 and + receives `X-FIP-Origin-Verify` as a CloudFront custom origin header. + +## Explicit deployment states + +The same template supports both states through `OriginLockdownEnabled`. + +| Contract | Deployment A (`false`) | Deployment B (`true`) | +|---|---|---| +| ALB SG port 80 | `0.0.0.0/0` | CloudFront origin prefix list only | +| ALB default action | forward to frontend target group | fixed HTTP 403 | +| Header rule | matching token forwards | matching token forwards | +| Prefix-list parameter | may be empty | required by template Rule | +| Cognito deletion protection | `INACTIVE` for rollback-safe first creation | `ACTIVE` after verified lockdown | + +The public and prefix-list ingress rules are conditional alternatives; there is +no intended steady state in which both are active. Deployment B must not proceed +unless the current managed prefix-list ID, weight and ALB-SG quota are rechecked. + +## Origin verification token handling + +`OriginVerificationToken` is `NoEcho`, accepts only base64url text of length +43-128, and must be generated at deployment time from at least 32 bytes of +cryptographic entropy. Never commit, print, log, output, embed in the frontend, +or place a real value in the example parameter file. + +`NoEcho` only redacts ordinary CloudFormation presentation. Principals with +sufficient CloudFront/ELB/configuration permissions can inspect relevant +configuration. Treat the value as a rotatable origin-verification token, not as +a permanently unknowable secret. + +For a controlled rotation without losing customer reachability: + +1. Create and review an update to State A; execute only after approval. +2. Generate a new token without printing it, update the State-A stack, and wait + for CloudFront to report `Deployed` and the ECS service to remain stable. +3. Re-resolve the prefix list and quota, create/review State B, and execute only + after approval. + +This procedure temporarily restores public ALB access and therefore requires a +reviewed maintenance window. Do not attempt an unreviewed single-step rotation +in locked State B. + +## Deployment A procedure (future authorized phase only) + +Deployment A has a hard database prerequisite. First create and review the +separate `db-access-prerequisite.yaml` stack. Do not execute it without the +exact `EXECUTE DB ACCESS PREREQUISITE` gate. After execution is separately +approved, its one-off task must configure and verify the writer on the existing +RDS database before `ApplicationDatabaseWriterSecretArn` is supplied to this +main stack. Creating the prerequisite stack does not run the task or update the +application service. See `database-access-architecture.md` for the exact grants +and rollback boundary. + +Deployment A uses `OriginLockdownEnabled=false`. It may create CloudFront, +Cognito, the three groups, origin policies, managed-login branding and the +origin-header listener rule; update the ECS task role/definition/service and +outputs; and retain public ALB port 80 plus the listener's default forward. +Cognito deletion protection is `INACTIVE` in this rollback-sensitive creation +state and becomes `ACTIVE` only with approved Deployment B. + +Before a change set, recheck identity, account, region, stack stability, branch, +HEAD, worktree, exact image digests, and Cognito-domain-prefix availability. +Generate the real token only in that authorized phase. Create and inspect the +change set with `CAPABILITY_IAM`, but never execute it without the exact approval +required by the controlling deployment procedure. + +Stop if the proposed update deletes or replaces RDS, VPC, ALB, ECS cluster, +target group or import bucket, changes the database unexpectedly, or includes a +data migration. + +After an approved execution and stable stack, create the bootstrap user through +the normal Cognito invitation workflow without setting or displaying a static +password. Assign exactly the `admin` app group and verify the user is enabled +with no other app group. The invited administrator then manually verifies the +temporary-password/new-password/TOTP/PKCE login and logout flow. + +Deployment A smoke gates are CloudFront HTTPS reachability, HTTP-to-HTTPS +redirect, safe `/api/auth/config`, unauthenticated protected API HTTP 401, +Managed Login reachability, exact callback/logout URLs, valid access-token +forwarding, correct `/api/auth/me`, and customer/operator/admin backend checks. +Direct ALB access remains expected in State A. + +## Deployment B procedure (future authorized phase only) + +Deployment B is allowed only after the user explicitly confirms the successful +manual login, logout and role behavior. Re-resolve +`com.amazonaws.global.cloudfront.origin-facing` in `eu-west-1`, verify its +current weight plus the ALB-SG quota, and pass that ID with +`OriginLockdownEnabled=true`. + +The reviewed change set should only replace the public ALB ingress with the +prefix-list ingress and change the default listener action to fixed 403. It must +not replace protected resources or alter data. After separately approved +execution, direct ALB requests without the origin header must be denied while +CloudFront and authenticated APIs remain healthy. + +## Data and runtime safety retained + +The application task remains Fargate Linux/ARM64 and non-root. Nginx listens on +8080 and proxies `/api` to `127.0.0.1:8000`. PostgreSQL uses the restricted +application role and read-only transaction defaults for customer-serving +access. RDS remains private, encrypted and TLS-required. The established +versioned SQLite-to-PostgreSQL migration, reconciliation, materialization and +atomic dataset activation workflow is unchanged and is not run by either +security deployment state. + +The template continues to use the existing stack resources and retains the +private import bucket, isolated RDS subnets, log groups, task definitions and +migration controls. Image tags must be immutable full-SHA tags and resolved to +digests before any future change set. + +## Cost and operational caveats + +The existing Fargate, ALB, RDS, public IPv4, secrets and log costs remain the +dominant monthly spend. CloudFront adds request/data-transfer charges and +Cognito adds monthly-active-user charges above applicable service allowances; +both depend on pilot traffic. The previous low-traffic estimate must be refreshed +with the AWS Pricing Calculator before execution because prices and usage are +time-dependent. Add budgets/alarms before broader distribution. + +No WAF, custom domain, end-to-end TLS, Multi-AZ database, private ECS egress or +production availability claim is introduced by this pilot design. diff --git a/docs/remediation/database-access-architecture.md b/docs/remediation/database-access-architecture.md new file mode 100644 index 0000000..3be42ce --- /dev/null +++ b/docs/remediation/database-access-architecture.md @@ -0,0 +1,162 @@ +# Runtime PostgreSQL access architecture + +Status: implemented local release gate. AWS role provisioning remains pending the +separate `EXECUTE DB ACCESS PREREQUISITE` approval. + +## Principals and secret boundaries + +| Principal | Runtime use | Database rights | Secret availability | +|---|---|---|---| +| Reader (`foundation_app`) | Startup pool construction, readiness and every normal read | `CONNECT`, schema `USAGE`, explicit table `SELECT`; default transactions read-only | Application task and migration/release task | +| Writer (`foundation_app_writer`) | Explicitly authorized operator/admin mutations, workers and durable idempotency | `CONNECT`, schema `USAGE`, explicit read/DML/column allowlists below | Application task only after the prerequisite succeeds; prerequisite task | +| Migration/owner (`foundation_admin`) | Alembic, role/grant maintenance and idempotent static bootstrap | Existing privileged RDS migration/owner rights | Migration/release and DB-prerequisite execution roles only | + +The application task execution role cannot read the RDS master secret. Missing +writer configuration fails closed with HTTP 503 for a mutation; it never falls +back to reader or migration credentials. Reader readiness remains independent +of writer availability. + +## Startup and GET side-effect audit + +| Code path | Trigger | Prior operation/table | Reason | Required at startup? | New target | +|---|---|---|---|---|---| +| `bff.main.lifespan` -> `PipelineRepository.synchronize_sources` | Every non-public PostgreSQL startup | `INSERT ... ON CONFLICT DO UPDATE`, `source_configurations` | Persist code-owned source defaults | No | Explicit migration/release bootstrap; insert missing rows only | +| `bff.main.lifespan` -> `GovernanceRepository.synchronize_policies` | Every non-public PostgreSQL startup | `INSERT ... ON CONFLICT DO UPDATE`, `retention_policies` | Persist policy defaults | No | Explicit migration/release bootstrap; insert missing rows only | +| Request audit middleware -> `PostgresAuditSink` | Authenticated request, including GET | `INSERT`, `audit_events` | Durable request audit | No database side effect is acceptable for GET | Structured application logs; no runtime DB grant | +| Readiness and liveness | Health probe | `SELECT` only / no SQL | Runtime health | Yes | Reader pool only | +| Repository construction and dependency resolution | Import/startup/request | None | Construct lazy engines/session factories | Yes | Reader by default; writer only in explicit mutation dependencies | + +There are no module-import, startup, lifespan, dependency-initialization, +liveness or readiness DML operations. Cache hydration, funder enrichment and +last-access persistence are not performed by GET handlers. The news endpoints +may call an external provider when separately configured, but do not persist a +database mutation. + +## Productive mutation matrix + +All listed idempotent HTTP mutations additionally use the writer-backed +`idempotency_records` store (`SELECT`, `INSERT`, `UPDATE`, `DELETE`). Backend +RBAC is evaluated before the writer dependency. + +| Endpoint / job | App role | Repository operation | Tables read | Tables written | SQL privileges/session | +|---|---|---|---|---|---| +| `POST /api/charities/directory/organizations/enrich` | operator | `PostgresJobRepository.enqueue` | `dataset_versions`, `job_runs` | `job_runs`, `job_dispatch_outbox`, `job_events` | writer: `SELECT`, `INSERT` | +| `POST /api/charities/grants/funders/enrich` | operator | `PostgresJobRepository.enqueue` | `dataset_versions`, `job_runs` | `job_runs`, `job_dispatch_outbox`, `job_events` | writer: `SELECT`, `INSERT` | +| `POST /api/charities/grants/funders/{key}/profile-cache` | operator | `SourceFunderRepository.queue_profile_cache` | `dataset_versions`, `grant_source_funder_facts`, `source_funder_link_overrides`, `source_funder_profile_cache`, `job_runs` | `job_runs`, `source_funder_profile_cache` | writer: allowlisted `SELECT`, `INSERT`, `UPDATE` | +| `POST /api/charities/grants/funders/{key}/reset-to-observed` | admin | `SourceFunderRepository.reset` | `dataset_versions`, `grant_source_funder_facts`, `source_funder_link_overrides` | `source_funder_link_overrides`, `source_funder_profile_cache` | writer: `SELECT`, `INSERT`, `UPDATE`, `DELETE` | +| `POST /api/charities/grants/funders/{key}/relink` | admin | `SourceFunderRepository.relink` | previous set plus `charities` | `source_funder_link_overrides`, `source_funder_profile_cache` | writer: allowlisted DML | +| `POST /api/admin/pipeline/trigger` | operator | `PostgresJobRepository.enqueue` | `dataset_versions`, `job_runs` | `job_runs`, `job_dispatch_outbox`, `job_events` | writer: `SELECT`, `INSERT` | +| `PUT /api/admin/pipeline/sources/{name}/schedule` | admin | `PipelineRepository.set_source_enabled` | `source_configurations` | `source_configurations.enabled`, `.updated_at` | writer: `SELECT`, column `UPDATE` only | +| `POST /api/admin/governance/retention/dry-run` | admin | `active_holds`, `record_retention_plan` | `data_holds` | `retention_actions`, `deletion_manifests` | writer: `SELECT`, `INSERT` | +| `POST /api/admin/governance/holds` | admin | `create_hold` | none | `data_holds` | writer: `INSERT` | +| `POST /api/admin/governance/holds/{id}/release` | admin | `release_hold` | `data_holds` | `data_holds` | writer: `SELECT`, `UPDATE` | +| `POST /api/admin/governance/data-subject-requests` | admin | `create_data_subject_request` | none | `data_subject_requests` | writer: `INSERT` | +| Durable worker claim/heartbeat/succeed/fail/requeue | internal operator worker | `PostgresJobRepository` worker methods | `job_runs`, `job_events` | `job_runs`, `job_events`, `worker_heartbeats` | writer: allowlisted `SELECT`, `INSERT`, `UPDATE` | +| Durable outbox publisher | internal operator worker | dispatch methods | `job_dispatch_outbox` | `job_dispatch_outbox` | writer: `SELECT`, `UPDATE` | +| Source ingestion worker | internal operator worker | `PipelineRepository` ingestion methods | `dataset_versions`, `source_configurations`, `source_ingestion_runs` | `source_ingestion_runs`, `storage_objects`, `ingestion_run_manifests` | writer: allowlisted `SELECT`, `INSERT`, `UPDATE` | +| Restore evidence | explicit release/governance worker | `record_restore_verification` | none | `restore_verifications` | writer: `INSERT` | + +`POST /api/charities/{number}/score` is a pure calculation and remains on the +reader. Cognito user management changes Cognito only and receives no database +session. Static source/policy synchronization is not reachable from an HTTP +route or normal worker startup; it is release-path bootstrap code. + +## Exact database allowlists + +Reader `SELECT` tables: + +```text +alembic_version +analytics_country_aggregates +analytics_country_connections +analytics_country_funder_rankings +analytics_entity_rankings +analytics_filter_values +analytics_funder_relationships +analytics_period_aggregates +analytics_programme_aggregates +analytics_scope_totals +charities +charity_registry_organizations +data_holds +dataset_versions +export_jobs +grant_beneficiary_countries +grant_overview_facts +grant_programme_categories +grant_source_funder_facts +grants +job_dispatch_outbox +job_events +job_runs +materialization_versions +organization_registry_links +retention_policies +source_configurations +source_funder_link_overrides +source_funder_profile_cache +source_ingestion_runs +``` + +Writer `SELECT` tables: + +```text +charities +data_holds +dataset_versions +grant_source_funder_facts +idempotency_records +job_dispatch_outbox +job_events +job_runs +source_configurations +source_funder_link_overrides +source_funder_profile_cache +source_ingestion_runs +worker_heartbeats +``` + +Writer table DML: + +| Table | Privileges | +|---|---| +| `data_holds` | `INSERT`, `UPDATE` | +| `data_subject_requests` | `INSERT` | +| `deletion_manifests` | `INSERT` | +| `idempotency_records` | `INSERT`, `UPDATE`, `DELETE` | +| `ingestion_run_manifests` | `INSERT` | +| `job_dispatch_outbox` | `INSERT`, `UPDATE` | +| `job_events` | `INSERT` | +| `job_runs` | `INSERT`, `UPDATE` | +| `restore_verifications` | `INSERT` | +| `retention_actions` | `INSERT` | +| `source_funder_link_overrides` | `INSERT`, `UPDATE` | +| `source_funder_profile_cache` | `INSERT`, `UPDATE`, `DELETE` | +| `source_ingestion_runs` | `INSERT`, `UPDATE` | +| `storage_objects` | `INSERT` | +| `worker_heartbeats` | `INSERT`, `UPDATE` | +| `source_configurations` | column-level `UPDATE(enabled, updated_at)` | + +No runtime sequence privilege is required: runtime identifiers are UUIDs and +the identity-backed dataset version is release-owned. No blanket current or +default table DML/SELECT grant is used. Provisioning first demotes role +attributes, revokes role memberships and existing database/schema/table/column/ +sequence/function/default privileges, then applies the exact allowlists. + +## Prerequisite order and rollback boundary + +1. Create and review the separate DB-access prerequisite stack. It may create + only the generated writer secret, log group, one-off ARM64 task definition + and its narrowly scoped roles. +2. Do not update the application service. The existing task continues with the + reader secret and existing image. +3. Only after the exact execute gate, execute the prerequisite stack and then + explicitly run its task in the existing private ECS/RDS network. +4. The task acquires an advisory lock, inserts only missing static defaults, + configures exact reader/writer grants and verifies both principals. It does + not migrate datasets or change schema. +5. Only after successful AWS permission evidence may the main stack receive the + writer secret ARN and a new application image. + +The prerequisite template does not contain an ECS service, RDS, VPC, ALB, +CloudFront or Cognito resource and cannot perform a service cutover by itself. diff --git a/docs/remediation/environment-variable-reference.md b/docs/remediation/environment-variable-reference.md index edc834e..c8b4d51 100644 --- a/docs/remediation/environment-variable-reference.md +++ b/docs/remediation/environment-variable-reference.md @@ -14,6 +14,11 @@ | `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_NAME`, `DATABASE_USER` | Alternative | PostgreSQL endpoint components | | `DATABASE_PASSWORD_FILE` | Preferred | Mounted runtime-only secret file | | `DATABASE_PASSWORD` | Alternative | Runtime-injected secret; never log/commit | +| `DATABASE_WRITE_URL` | Writer alternative | Dedicated writer URL; never falls back to reader credentials | +| `DATABASE_WRITE_HOST`, `DATABASE_WRITE_PORT`, `DATABASE_WRITE_NAME` | Writer optional | May override reader network/database values | +| `DATABASE_WRITE_USER`, `DATABASE_WRITE_PASSWORD_FILE`, `DATABASE_WRITE_PASSWORD` | Writer mutation path | Dedicated restricted DML identity; credentials are mandatory as a complete set | +| `DATABASE_WRITE_SSL_MODE` | No | Defaults to reader SSL mode; `require` in AWS | +| `DATABASE_WRITE_POOL_SIZE`, `DATABASE_WRITE_MAX_OVERFLOW` | No | Independently bounded writer pool | | `DATABASE_POOL_SIZE`, `DATABASE_MAX_OVERFLOW` | No | Positive bounded pool values | | `DATABASE_POOL_TIMEOUT_SECONDS` | No | Positive checkout timeout | | `DATABASE_CONNECT_TIMEOUT_SECONDS` | No | Positive connection/readiness timeout | diff --git a/frontend/e2e/phase7.spec.ts b/frontend/e2e/phase7.spec.ts index b6240ae..e22206d 100644 --- a/frontend/e2e/phase7.spec.ts +++ b/frontend/e2e/phase7.spec.ts @@ -75,6 +75,7 @@ const overview = { }; const responses = { + auth: { mode: "public_readonly" }, health: { status: "ok" }, stats: { total_charities: 42, @@ -103,7 +104,8 @@ async function installApiRoutes(page: Page) { const key = `${url.pathname}${url.search}`; requestCounts.set(key, (requestCounts.get(key) || 0) + 1); let body: unknown; - if (url.pathname === "/api/charities/stats") body = responses.stats; + if (url.pathname === "/api/auth/config") body = responses.auth; + else if (url.pathname === "/api/charities/stats") body = responses.stats; else if (url.pathname === "/api/charities/grants/overview") body = responses.overview; else if (url.pathname === "/api/charities/grants/beneficiary-geographies") body = responses.geographies; else if (url.pathname === "/api/charities/grants/overview/entity-suggestions") body = responses.entitySuggestions; diff --git a/frontend/e2e/public-readonly.spec.ts b/frontend/e2e/public-readonly.spec.ts index 0cff6b6..61c8306 100644 --- a/frontend/e2e/public-readonly.spec.ts +++ b/frontend/e2e/public-readonly.spec.ts @@ -1,6 +1,80 @@ import { expect, test } from "@playwright/test"; + +const emptyMap = { + status: "available", + geographic_dimension: "beneficiary", + items: [], + known_geography_count: 0, + unknown_geography_count: 0, + coverage_percentage: 0, + currencies: ["EUR"], + selected_currency: "EUR", + funding_status: "available", + funding_mode_available: true, + grant_country_association_count: 0, + multi_country_grant_count: 0, + funding_excluded_multi_country_count: 0, + funding_excluded_multi_country_amount: 0, + funding_excluded_currency_count: 0, + funding_excluded_invalid_amount_count: 0, + connections: [], + connection_grant_count: 0, + connection_excluded_no_headquarters_count: 0, + connection_same_country_count: 0, + minimum_coverage_threshold: 0, + metadata: { data_mode: "public_readonly_browser_test", source: ["fixture"], record_count: 0, limitations: [] }, +}; + +const emptyOverview = { + status: "available", + kpis: { + awarded_funding: 0, + currency: "EUR", + grants_monitored: 0, + country_coverage_percentage: 0, + mapped_grant_count: 0, + unmapped_grant_count: 0, + programme_coverage_percentage: 0, + classified_grant_count: 0, + qualifying_programme_grant_count: 0, + }, + map: emptyMap, + trends: { status: "available", currency: "EUR", available_currencies: ["EUR"], granularity: "monthly", period: null, items: [], excluded: {}, last_refreshed_at: null }, + themes: { status: "available", currency: "EUR", items: [], classification_coverage: { classified_percentage: 0, classified_grant_count: 0, unclassified_grant_count: 0 } }, + available_date_range: { from: null, to: null }, +}; + test("anonymous demo exposes the reviewed read-only UI routes", async ({ context, page }) => { + await page.route("**/api/**", async route => { + const request = route.request(); + const url = new URL(request.url()); + const mutation = request.method() !== "GET" && request.method() !== "HEAD"; + const protectedPath = url.pathname.startsWith("/api/admin/"); + if (mutation || protectedPath) { + await route.fulfill({ status: 401, json: { detail: "Authentication required." } }); + return; + } + if (url.pathname === "/api/auth/config") { + await route.fulfill({ status: 200, json: { mode: "public_readonly" } }); + return; + } + if (url.pathname === "/api/charities/grants/overview") { + await route.fulfill({ status: 200, json: emptyOverview }); + return; + } + if (url.pathname === "/api/charities/stats") { + await route.fulfill({ status: 200, json: { total_charities: 0, active_charities: 0, removed_charities: 0, average_income: 0, average_expenditure: 0, source: ["fixture"] } }); + return; + } + if (url.pathname === "/api/charities/grants/beneficiary-geographies") { + await route.fulfill({ status: 200, json: [] }); + return; + } + await route.fulfill({ status: 200, json: { items: [] } }); + }); + await page.route("**/health", route => route.fulfill({ status: 200, json: { status: "ok" } })); + const overviewResponsePromise = page.waitForResponse(response => { const url = new URL(response.url()); return url.pathname === "/api/charities/grants/overview" @@ -13,16 +87,22 @@ test("anonymous demo exposes the reviewed read-only UI routes", async ({ context expect(overviewResponse.headers()["set-cookie"]).toBeUndefined(); await expect(page.locator(".world-map-card")).toBeVisible(); - const adminResponse = await page.request.get("/api/admin/pipeline/status"); - expect(adminResponse.status()).toBe(401); - const directoryResponse = await page.request.get("/api/charities"); - expect(directoryResponse.status()).toBe(200); - expect(directoryResponse.headers()["set-cookie"]).toBeUndefined(); - const mutationResponse = await page.request.post("/api/admin/pipeline/trigger", { - data: { source: "quick_consolidate" }, - headers: { "Idempotency-Key": "public-demo-browser-test" }, + const results = await page.evaluate(async () => { + const admin = await fetch("/api/admin/pipeline/status", { credentials: "omit" }); + const directory = await fetch("/api/charities", { credentials: "omit" }); + const mutation = await fetch("/api/admin/pipeline/trigger", { + method: "POST", + credentials: "omit", + headers: { "Content-Type": "application/json", "Idempotency-Key": "public-demo-browser-test" }, + body: JSON.stringify({ source: "quick_consolidate" }), + }); + return { + admin: admin.status, + directory: directory.status, + directoryCookie: directory.headers.get("set-cookie"), + mutation: mutation.status, + }; }); - expect(mutationResponse.status()).toBe(401); - + expect(results).toEqual({ admin: 401, directory: 200, directoryCookie: null, mutation: 401 }); expect(await context.cookies()).toEqual([]); }); diff --git a/frontend/scripts/check-runtime-layout.mjs b/frontend/scripts/check-runtime-layout.mjs index 04db5cc..3d13fc3 100644 --- a/frontend/scripts/check-runtime-layout.mjs +++ b/frontend/scripts/check-runtime-layout.mjs @@ -170,6 +170,7 @@ const overview = { const browserMock = `(() => { const responses = { + auth: ${JSON.stringify({ mode: "public_readonly" })}, health: ${JSON.stringify({ status: "ok" })}, stats: ${JSON.stringify({ total_charities: 42, active_charities: 40, removed_charities: 2, average_income: 1000000, average_expenditure: 800000, source: ["360Giving", "Charity Commission for England and Wales", "Philea"] })}, overview: ${JSON.stringify(overview)}, @@ -184,7 +185,8 @@ const browserMock = `(() => { const key = url.pathname + url.search; window.__phase7Requests[key] = (window.__phase7Requests[key] || 0) + 1; let body; - if (url.pathname === "/health") body = responses.health; + if (url.pathname === "/api/auth/config") body = responses.auth; + else if (url.pathname === "/health") body = responses.health; else if (url.pathname === "/api/charities/stats") body = responses.stats; else if (url.pathname === "/api/charities/grants/overview") body = responses.overview; else if (url.pathname === "/api/charities/grants/beneficiary-geographies") body = responses.geographies; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0dd7ab8..67f4e50 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -25,6 +25,9 @@ import { PanelLeftOpen, Star, X, + Users, + LogIn, + LogOut, } from "lucide-react"; import AppHeader from "./components/AppHeader"; import type { @@ -41,7 +44,9 @@ import { grantScopeFromUrl, type GrantScope, } from "./lib/grantScope"; -import { mutationHeaders } from "./lib/http"; +import { apiFetch, mutationHeaders } from "./lib/http"; +import { avatarInitials } from "./lib/avatar"; +import { useAuth } from "./auth/authState"; import amplifyLogo from "./assets/amplify-logo.svg"; import type { GrantMapFilters, @@ -56,6 +61,7 @@ const ProgrammeAllocationChart = lazy(() => import("./components/DataCharts").th const FinancialHistoryChart = lazy(() => import("./components/DataCharts").then(module => ({ default: module.FinancialHistoryChart }))); const RegistryDirectory = lazy(() => import("./components/RegistryDirectory")); const DonorDirectoryPage = lazy(() => import("./components/DonorDirectoryPage")); +const UserManagementPage = lazy(() => import("./components/UserManagementPage")); // Configuration for API requests const API_BASE = import.meta.env.VITE_API_BASE_URL @@ -131,11 +137,6 @@ type FavoriteDonorWorkspace = | { kind: "donor"; item: FavoriteDonorPayload } | { kind: "request"; item: FavoriteDonorRequestPayload }; -type ActiveSourceFunder = { - sourceFunderKey: string; - displayName: string; -}; - type NewsSourceItem = { title: string; link: string; @@ -515,7 +516,7 @@ const INITIAL_PROFILE_LOADING: ProfileLoadingState = { source_record: profileSectionState("idle"), }; -type GlobalApiErrorKey = "health" | "statistics" | "directory" | "source_reset"; +type GlobalApiErrorKey = "health" | "statistics" | "directory"; type GlobalApiErrors = Partial>; function abortableDelay( @@ -560,6 +561,14 @@ interface PipelineStatus { error: string | null; } +interface PublicSourceStatus { + name: string; + enabled: boolean; + freshness: string; + last_success_at: string | null; + record_count: number | null; +} + // Fallback Mock Data for demo when BFF is offline const MOCK_STATS: KPIStats = { total_charities: 15, @@ -747,9 +756,21 @@ const EMPTY_GRANT_MAP_FILTERS: GrantMapFilters = { }; export default function App() { + const auth = useAuth(); + if (auth.loading) { + return
; + } + return ; +} + + +function AuthenticatedApplication() { + const auth = useAuth(); + const canOperate = auth.role === "operator" || auth.role === "admin"; + const isAdmin = auth.role === "admin"; const hasFunderRoute = () => Boolean(new URLSearchParams(window.location.search).get("funder_country")); const initialView = () => new URLSearchParams(window.location.search).get("view"); - const [activeTab, setActiveTab] = useState<"overview" | "directory" | "favorites" | "admin">(() => initialView() === "pipeline" ? "admin" : initialView() === "favorites" ? "favorites" : (hasFunderRoute() || ["donors", "research", "registry"].includes(initialView() || "")) ? "directory" : "overview"); + const [activeTab, setActiveTab] = useState<"overview" | "directory" | "favorites" | "admin" | "users">(() => initialView() === "users" && isAdmin ? "users" : initialView() === "pipeline" ? "admin" : initialView() === "favorites" ? "favorites" : (hasFunderRoute() || ["donors", "research", "registry"].includes(initialView() || "")) ? "directory" : "overview"); const [directoryMode, setDirectoryMode] = useState<"donors" | "profiles" | "registry">(() => initialView() === "research" ? "profiles" : initialView() === "registry" ? "registry" : "donors"); const [overviewFilterCount, setOverviewFilterCount] = useState(0); const [overviewFiltersOpen, setOverviewFiltersOpen] = useState(false); @@ -818,8 +839,6 @@ export default function App() { const sourceHydrationTimerRef = useRef(null); const selectedSourceFunderProfileKeyRef = useRef(null); const sourceFunderProfilePayloadsRef = useRef(new Map>()); - const [activeSourceFunder, setActiveSourceFunder] = useState(null); - const [sourceFunderResetPending, setSourceFunderResetPending] = useState(false); // News summarizer states const [newsLoading, setNewsLoading] = useState(false); @@ -894,6 +913,8 @@ export default function App() { const view = initialView(); if (view === "pipeline") { setActiveTab("admin"); + } else if (view === "users" && isAdmin) { + setActiveTab("users"); } else if (view === "favorites") { setActiveTab("favorites"); } else if (view === "research") { @@ -911,7 +932,7 @@ export default function App() { }; window.addEventListener("popstate", syncRouteMode); return () => window.removeEventListener("popstate", syncRouteMode); - }, []); + }, [isAdmin]); // The stats endpoint returns a fresh array on every response. Keep the // source list referentially stable while its values are unchanged so it @@ -1024,6 +1045,7 @@ export default function App() { error: null }); const [logs, setLogs] = useState("System terminal ready.\n"); + const [scraperSources, setScraperSources] = useState([]); const [isTriggering, setIsTriggering] = useState(false); const [pipelineLimit, setPipelineLimit] = useState(20); const [pipelineFresh, setPipelineFresh] = useState(false); @@ -1031,6 +1053,7 @@ export default function App() { const [pipelineSearch, setPipelineSearch] = useState(""); const [pipelineIds, setPipelineIds] = useState(""); const logsEndRef = useRef(null); + const authenticatedIdentity = auth.identity?.email || auth.identity?.username || ""; const runHealthCheck = useEffectEvent((signal: AbortSignal) => { void checkBffHealth(signal); @@ -1051,7 +1074,7 @@ export default function App() { void fetchCharityGrants(id, requestId, signal); void fetchSankeyData(id, requestId, signal); void fetchScoreData(id, requestId, signal); - if (sourceFunderKey) { + if (sourceFunderKey && canOperate) { void hydrateSourceFunderProfile(sourceFunderKey, id, requestId, signal); } }); @@ -1107,8 +1130,8 @@ export default function App() { useEffect(() => { if (!isBffOnline) return; const controller = new AbortController(); - fetch(`${API_BASE}/api/charities/grants/beneficiary-geographies`, { - credentials: "include", + apiFetch(`${API_BASE}/api/charities/grants/beneficiary-geographies`, { + credentials: "omit", signal: controller.signal, }) .then(async response => { @@ -1150,8 +1173,8 @@ export default function App() { } const parameters = new URLSearchParams({ search: query, limit: "6" }); parameters.set("sources", selectedDataSources.join(",")); - const response = await fetch(`${API_BASE}/api/charities?${parameters.toString()}`, { - credentials: "include", + const response = await apiFetch(`${API_BASE}/api/charities?${parameters.toString()}`, { + credentials: "omit", signal: controller.signal, }); if (!response.ok) throw new Error(`Organization suggestion request failed (${response.status}).`); @@ -1213,7 +1236,7 @@ export default function App() { setScoreData(null); setProfileLoading({ ...INITIAL_PROFILE_LOADING, - source_record: selectedSourceFunderProfileKey && isBffOnline + source_record: selectedSourceFunderProfileKey && isBffOnline && canOperate ? profileSectionState("loading") : profileSectionState("idle"), }); @@ -1232,7 +1255,7 @@ export default function App() { sourceHydrationTimerRef.current = null; } }; - }, [selectedCharityId, selectedSourceFunderProfileKey, isBffOnline]); + }, [selectedCharityId, selectedSourceFunderProfileKey, isBffOnline, canOperate]); useEffect(() => { if (activeTab !== "admin") return; @@ -1296,26 +1319,10 @@ export default function App() { return () => window.removeEventListener("registry-header-state", receiveRegistryHeaderState); }, []); - useEffect(() => { - const receiveActiveSourceFunder = (event: Event) => { - const detail = (event as CustomEvent).detail; - if (!detail?.sourceFunderKey) { - setActiveSourceFunder(null); - return; - } - setActiveSourceFunder({ - sourceFunderKey: detail.sourceFunderKey, - displayName: detail.displayName || "Source funder", - }); - }; - window.addEventListener("active-source-funder-change", receiveActiveSourceFunder); - return () => window.removeEventListener("active-source-funder-change", receiveActiveSourceFunder); - }, []); - const checkBffHealth = async (signal?: AbortSignal) => { setInitialLoading(true); try { - const resp = await fetch(`${API_BASE}/health`, { signal }); + const resp = await apiFetch(`${API_BASE}/health`, { signal }); if (signal?.aborted) return; if (resp.ok) { setAuthError(null); @@ -1340,8 +1347,8 @@ export default function App() { const isOnline = forceOnline !== undefined ? forceOnline : isBffOnline; if (!isOnline) return; try { - const resp = await fetch(`${API_BASE}/api/charities/stats`, { - credentials: "include", + const resp = await apiFetch(`${API_BASE}/api/charities/stats`, { + credentials: "omit", signal, }); if (signal?.aborted) return; @@ -1446,7 +1453,7 @@ export default function App() { url += `&max_avg_grant_size=${maxAvgGrant}`; } - const resp = await fetch(url, { credentials: "include", signal: controller.signal }); + const resp = await apiFetch(url, { credentials: "omit", signal: controller.signal }); if (requestSequence !== directoryRequestSequenceRef.current) return; if (resp.ok) { const data: Charity[] = await resp.json(); @@ -1506,8 +1513,8 @@ export default function App() { } return; } - const resp = await fetch(`${API_BASE}/api/charities/${id}`, { - credentials: "include", + const resp = await apiFetch(`${API_BASE}/api/charities/${id}`, { + credentials: "omit", signal, }); if (!isCurrentProfileRequest(requestId)) return; @@ -1560,11 +1567,11 @@ export default function App() { ); setProfileSection(requestId, "source_record", "loading"); try { - const queued = await fetch( + const queued = await apiFetch( `${API_BASE}/api/charities/grants/funders/${encodeURIComponent(sourceFunderKey)}/profile-cache`, { method: "POST", - credentials: "include", + credentials: "omit", headers: mutationHeaders("hydrate source-funder profile"), signal, }, @@ -1576,9 +1583,9 @@ export default function App() { } for (let attempt = 0; attempt <= 20; attempt += 1) { - const response = await fetch( + const response = await apiFetch( `${API_BASE}/api/charities/grants/funders/${encodeURIComponent(sourceFunderKey)}/profile-cache`, - { credentials: "include", signal }, + { credentials: "omit", signal }, ); if (!requestIsCurrent()) return; @@ -1695,6 +1702,7 @@ export default function App() { }; const fetchFoundationNews = async (organization: Charity) => { + if (!canOperate) return; const name = organization.charity_name; const organizationKey = newsOrganizationKey(organization); newsAbortControllerRef.current?.abort(); @@ -1762,8 +1770,8 @@ export default function App() { query.set("lookback", "all"); } const suffix = query.size ? `?${query.toString()}` : ""; - const resp = await fetch(`${API_BASE}/api/news/${encodeURIComponent(name)}/summary/stream${suffix}`, { - credentials: "include", + const resp = await apiFetch(`${API_BASE}/api/news/${encodeURIComponent(name)}/summary/stream${suffix}`, { + credentials: "omit", headers: { Accept: "text/event-stream" }, signal: controller.signal, }); @@ -1863,9 +1871,9 @@ export default function App() { } const query = params.toString(); try { - const resp = await fetch( + const resp = await apiFetch( `${API_BASE}/api/charities/grants/map${query ? `?${query}` : ""}`, - { credentials: "include", signal: controller.signal }, + { credentials: "omit", signal: controller.signal }, ); if (controller.signal.aborted || mapRequestRef.current !== controller) return; if (resp.ok) { @@ -1887,7 +1895,7 @@ export default function App() { }; const navigateApplication = ( - view: "overview" | "donors" | "research" | "registry" | "favorites" | "pipeline", + view: "overview" | "donors" | "research" | "registry" | "favorites" | "pipeline" | "users", mode: "push" | "replace" = "push", ) => { const query = new URLSearchParams(window.location.search); @@ -1899,6 +1907,8 @@ export default function App() { query.set("view", view); if (view === "pipeline") { setActiveTab("admin"); + } else if (view === "users" && isAdmin) { + setActiveTab("users"); } else if (view === "favorites") { setActiveTab("favorites"); } else { @@ -2188,12 +2198,12 @@ export default function App() { : ""; try { const [trendsResponse, themesResponse] = await Promise.all([ - fetch(`${API_BASE}/api/charities/grants/trends?months=24${currencyQuery}`, { - credentials: "include", + apiFetch(`${API_BASE}/api/charities/grants/trends?months=24${currencyQuery}`, { + credentials: "omit", signal: controller.signal, }), - fetch(`${API_BASE}/api/charities/grants/themes?${currencyQuery.slice(1)}`, { - credentials: "include", + apiFetch(`${API_BASE}/api/charities/grants/themes?${currencyQuery.slice(1)}`, { + credentials: "omit", signal: controller.signal, }) ]); @@ -2232,8 +2242,8 @@ export default function App() { } return; } - const resp = await fetch(`${API_BASE}/api/charities/${id}/grants`, { - credentials: "include", + const resp = await apiFetch(`${API_BASE}/api/charities/${id}/grants`, { + credentials: "omit", signal, }); if (!isCurrentProfileRequest(requestId)) return; @@ -2273,8 +2283,8 @@ export default function App() { } return; } - const resp = await fetch(`${API_BASE}/api/charities/${id}/sankey`, { - credentials: "include", + const resp = await apiFetch(`${API_BASE}/api/charities/${id}/sankey`, { + credentials: "omit", signal, }); if (!isCurrentProfileRequest(requestId)) return; @@ -2319,9 +2329,9 @@ export default function App() { if (isCurrentProfileRequest(requestId)) setScoreData(null); return; } - const resp = await fetch(`${API_BASE}/api/charities/${id}/score`, { + const resp = await apiFetch(`${API_BASE}/api/charities/${id}/score`, { method: "GET", - credentials: "include", + credentials: "omit", signal, }); if (!isCurrentProfileRequest(requestId)) return; @@ -2366,7 +2376,6 @@ export default function App() { setNewsLoading(false); setNewsProgressStep(null); setNewsRunMode(null); - setApiError("source_reset", null); const previousFocus = profilePreviousFocusRef.current; profilePreviousFocusRef.current = null; window.requestAnimationFrame(() => previousFocus?.focus()); @@ -2385,70 +2394,41 @@ export default function App() { return () => document.removeEventListener("keydown", closeOnEscape); }, [selectedCharity]); - const resetActiveSourceFunderToObserved = async () => { - const target = activeSourceFunder; - if (!target || !isBffOnline) { - // The NL control is also available while viewing an ordinary profile. - // In that case there is no source link to mutate, but its latest local - // news research still belongs to the profile being safely cleared. - cancelNewsResearch(selectedNewsOrganizationKeyRef.current, true); - clearActiveProfileSafely(); - return; - } - - setSourceFunderResetPending(true); - try { - const response = await fetch( - `${API_BASE}/api/charities/grants/funders/${encodeURIComponent(target.sourceFunderKey)}/reset-to-observed`, - { - method: "POST", - credentials: "include", - headers: mutationHeaders("reset source funder to observed-only"), - }, - ); - const body = await response.json(); - if (!response.ok) { - throw new Error(body.detail || `Could not reset ${target.displayName} to observed-only (${response.status}).`); - } - window.dispatchEvent(new CustomEvent("source-funder-reset-to-observed", { - detail: { sourceFunderKey: target.sourceFunderKey }, - })); - sourceFunderProfilePayloadsRef.current.delete(target.sourceFunderKey); - // News briefings are derived from this profile link and are stored only - // in this browser. Reset removes the matching latest briefing as well - // as cancelling a search that could otherwise finish after the reset. - cancelNewsResearch(selectedNewsOrganizationKeyRef.current, true); - setApiError("source_reset", null); - setActiveSourceFunder(null); - clearActiveProfileSafely(); - } catch (error) { - setApiError("source_reset", (error as Error).message || `Could not reset ${target.displayName} to observed-only.`); - } finally { - setSourceFunderResetPending(false); - } - }; - const fetchPipelineStatus = async (signal?: AbortSignal) => { if (!isBffOnline) return; try { - const resp = await fetch(`${API_BASE}/api/admin/pipeline/status`, { - credentials: "include", + const statusPath = canOperate ? "/api/admin/pipeline/status" : "/api/scraper/status"; + const resp = await apiFetch(`${API_BASE}${statusPath}`, { + credentials: "omit", signal, }); if (signal?.aborted) return; if (resp.ok) { const data = await resp.json(); - setPipelineStatus(data); + if (canOperate) { + setPipelineStatus(data); + } else { + setPipelineStatus({ + status: data.status, + started_at: data.last_run, + finished_at: data.last_run, + last_run_source: null, + error: null, + }); + setScraperSources(Array.isArray(data.sources) ? data.sources : []); + } } - const logResp = await fetch(`${API_BASE}/api/admin/pipeline/logs`, { - credentials: "include", - signal, - }); - if (signal?.aborted) return; - if (logResp.ok) { - const logData = await logResp.json(); - setLogs(logData.logs || "System idle.\n"); + if (isAdmin) { + const logResp = await apiFetch(`${API_BASE}/api/admin/pipeline/logs`, { + credentials: "omit", + signal, + }); + if (signal?.aborted) return; + if (logResp.ok) { + const logData = await logResp.json(); + setLogs(logData.logs || "System idle.\n"); + } } } catch (e) { if ((e as Error).name === "AbortError") return; @@ -2457,6 +2437,7 @@ export default function App() { }; const triggerPipeline = async (source: string) => { + if (!canOperate) return; setIsTriggering(true); // Enforce max limit of 100 on full runs @@ -2484,7 +2465,7 @@ export default function App() { return; } try { - const resp = await fetch(`${API_BASE}/api/admin/pipeline/trigger`, { + const resp = await apiFetch(`${API_BASE}/api/admin/pipeline/trigger`, { method: "POST", headers: mutationHeaders(`trigger ${source} pipeline`, true), body: JSON.stringify({ @@ -2495,7 +2476,7 @@ export default function App() { reg_numbers: regNumbers && regNumbers.length > 0 ? regNumbers : undefined, skip_contact_crawler: !enableImpressum }), - credentials: "include" + credentials: "omit" }); if (resp.ok) { const data = await resp.json(); @@ -2646,26 +2627,36 @@ export default function App() { title="Pipeline Monitor" > - Pipeline Monitor + {canOperate ? "Pipeline Monitor" : "Scraper Status"} + {isAdmin && }
- +
- Netlight Guest - Administrator + {auth.authenticated + ? auth.identity?.email || auth.identity?.username || "Authenticated user" + : "Netlight Guest"} + {auth.authenticated ? auth.role : "Not signed in"} + {auth.mode === "cognito_rbac" && (auth.authenticated + ? + : )}
+ {auth.error &&
{auth.error}
} {activeTab === "admin" &&
}> + + + )} + {/* Retained for rollback only; the active Overview is the compact, globally-filtered dashboard above. */} {SHOW_LEGACY_OVERVIEW && activeTab === "overview" && (
@@ -3043,6 +3041,7 @@ export default function App() { { @@ -3091,6 +3090,7 @@ export default function App() { key={directoryHandoff.version} apiBase={API_BASE} online={isBffOnline} + canOperate={canOperate} initialQuery={directoryHandoff.query} initialBeneficiaryGeography="" onOpenEnrichedProfile={(id, name) => { @@ -3433,7 +3433,7 @@ export default function App() {
{/* Trigger Buttons */} -
+ {canOperate &&

Predefined Pipeline Controls

Safely trigger the execution of scraping, crawling, or database reload operations. @@ -3445,7 +3445,7 @@ export default function App() {

-
+
} {/* Status Indicator Panel */}
@@ -3542,7 +3542,7 @@ export default function App() {
Pipeline State: - + {pipelineStatus.status}
@@ -3562,6 +3562,12 @@ export default function App() { Error: {pipelineStatus.error}
)} + {!canOperate && scraperSources.map(source => ( +
+ {source.name}{source.last_success_at ? `Last success ${new Date(source.last_success_at).toLocaleString()}` : "No successful run recorded"} + {source.freshness}{source.record_count !== null ? ` · ${source.record_count.toLocaleString()} records` : ""} +
+ ))}
@@ -3619,7 +3625,7 @@ export default function App() { {/* Logs output window */} -
+ {isAdmin &&
@@ -3631,11 +3637,11 @@ export default function App() {
-
+
{logs}
-
+
}
)} @@ -3741,14 +3747,14 @@ export default function App() { Download PDF )} - + } diff --git a/frontend/src/auth/AuthContext.tsx b/frontend/src/auth/AuthContext.tsx new file mode 100644 index 0000000..f0a1862 --- /dev/null +++ b/frontend/src/auth/AuthContext.tsx @@ -0,0 +1,242 @@ +import { + type ReactNode, + useEffect, + useRef, + useState, +} from "react"; + +import { setAccessTokenProvider } from "../lib/http"; +import { AuthContext, type AuthValue, type Identity } from "./authState"; + + +type AuthConfig = { + mode: string; + region?: string; + user_pool_id?: string; + client_id?: string; + domain?: string; + scopes?: string[]; +}; + +type TokenSet = { + access_token: string; + id_token?: string; + refresh_token?: string; + expires_in?: number; + token_type?: string; +}; + +const TOKEN_KEY = "fip-cognito-token-session"; +const STATE_KEY = "fip-cognito-oauth-state"; +const VERIFIER_KEY = "fip-cognito-pkce-verifier"; + + +function randomBase64Url(byteLength: number): string { + const bytes = new Uint8Array(byteLength); + crypto.getRandomValues(bytes); + let binary = ""; + bytes.forEach(value => { binary += String.fromCharCode(value); }); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, ""); +} + + +async function sha256Base64Url(value: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + let binary = ""; + new Uint8Array(digest).forEach(byte => { binary += String.fromCharCode(byte); }); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/g, ""); +} + + +function decodeDisplayClaims(token?: string): Record { + if (!token) return {}; + try { + const payload = token.split(".")[1]; + if (!payload) return {}; + const normalized = payload.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, "="); + return JSON.parse(atob(padded)) as Record; + } catch { + return {}; + } +} + + +function readTokens(): TokenSet | null { + try { + const raw = sessionStorage.getItem(TOKEN_KEY); + if (!raw) return null; + const parsed = JSON.parse(raw) as TokenSet; + return typeof parsed.access_token === "string" ? parsed : null; + } catch { + return null; + } +} + + +function tokenIsUsable(token: string): boolean { + const claims = decodeDisplayClaims(token); + return typeof claims.exp === "number" && claims.exp > Date.now() / 1000 + 60; +} + + +async function tokenRequest(config: AuthConfig, body: URLSearchParams): Promise { + if (!config.domain) throw new Error("Managed login domain is unavailable."); + const response = await fetch(`${config.domain}/oauth2/token`, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body, + }); + if (!response.ok) throw new Error("Managed login token exchange failed."); + const tokens = await response.json() as TokenSet; + if (!tokens.access_token) throw new Error("Managed login returned no access token."); + return tokens; +} + + +export function AuthProvider({ children }: { children: ReactNode }) { + const [loading, setLoading] = useState(true); + const [config, setConfig] = useState({ mode: "loading" }); + const [identity, setIdentity] = useState(null); + const [error, setError] = useState(null); + const initialized = useRef(false); + const tokenRef = useRef(null); + + const applyTokens = (next: TokenSet | null) => { + tokenRef.current = next; + if (next) sessionStorage.setItem(TOKEN_KEY, JSON.stringify(next)); + else sessionStorage.removeItem(TOKEN_KEY); + }; + + useEffect(() => { + setAccessTokenProvider(() => tokenRef.current?.access_token || null); + return () => setAccessTokenProvider(null); + }, []); + + useEffect(() => { + if (initialized.current) return; + initialized.current = true; + const initialize = async () => { + try { + const response = await fetch("/api/auth/config", { credentials: "omit" }); + if (!response.ok) throw new Error("Authentication configuration is unavailable."); + const loadedConfig = await response.json() as AuthConfig; + setConfig(loadedConfig); + if (loadedConfig.mode !== "cognito_rbac") { + setIdentity({ + sub: loadedConfig.mode === "public_readonly" ? "public-readonly-demo" : "local-user", + username: loadedConfig.mode === "public_readonly" ? "Public demo" : "Local user", + roles: [loadedConfig.mode === "public_readonly" ? "customer" : "admin"], + }); + return; + } + if (!loadedConfig.client_id || !loadedConfig.domain) { + throw new Error("Managed login configuration is incomplete."); + } + + let currentTokens = readTokens(); + if (window.location.pathname === "/auth/callback") { + const query = new URLSearchParams(window.location.search); + const code = query.get("code"); + const returnedState = query.get("state"); + const expectedState = sessionStorage.getItem(STATE_KEY); + const verifier = sessionStorage.getItem(VERIFIER_KEY); + history.replaceState({}, "", "/"); + sessionStorage.removeItem(STATE_KEY); + sessionStorage.removeItem(VERIFIER_KEY); + if (query.get("error")) throw new Error("Managed login was not completed."); + if (!code || !returnedState || !expectedState || returnedState !== expectedState || !verifier) { + throw new Error("Managed login callback validation failed."); + } + currentTokens = await tokenRequest(loadedConfig, new URLSearchParams({ + grant_type: "authorization_code", + client_id: loadedConfig.client_id, + code, + redirect_uri: `${window.location.origin}/auth/callback`, + code_verifier: verifier, + })); + } else if (currentTokens && !tokenIsUsable(currentTokens.access_token) && currentTokens.refresh_token) { + const refreshed = await tokenRequest(loadedConfig, new URLSearchParams({ + grant_type: "refresh_token", + client_id: loadedConfig.client_id, + refresh_token: currentTokens.refresh_token, + })); + currentTokens = { ...currentTokens, ...refreshed, refresh_token: refreshed.refresh_token || currentTokens.refresh_token }; + } + + if (!currentTokens || !tokenIsUsable(currentTokens.access_token)) { + applyTokens(null); + return; + } + applyTokens(currentTokens); + const me = await fetch("/api/auth/me", { + headers: { Authorization: `Bearer ${currentTokens.access_token}` }, + credentials: "omit", + }); + if (!me.ok) throw new Error("Authenticated identity could not be verified."); + const verified = await me.json() as Identity; + const idClaims = decodeDisplayClaims(currentTokens.id_token); + setIdentity({ + ...verified, + email: typeof idClaims.email === "string" ? idClaims.email : undefined, + }); + } catch (caught) { + applyTokens(null); + setIdentity(null); + setError((caught as Error).message || "Authentication failed."); + } finally { + setLoading(false); + } + }; + void initialize(); + }, []); + + const login = async () => { + if (!config.client_id || !config.domain) { + setError("Managed login configuration is unavailable."); + return; + } + const verifier = randomBase64Url(64); + const state = randomBase64Url(32); + const challenge = await sha256Base64Url(verifier); + sessionStorage.setItem(VERIFIER_KEY, verifier); + sessionStorage.setItem(STATE_KEY, state); + const query = new URLSearchParams({ + response_type: "code", + client_id: config.client_id, + redirect_uri: `${window.location.origin}/auth/callback`, + scope: (config.scopes || ["openid", "email", "profile"]).join(" "), + state, + code_challenge: challenge, + code_challenge_method: "S256", + }); + window.location.assign(`${config.domain}/oauth2/authorize?${query.toString()}`); + }; + + const logout = () => { + applyTokens(null); + setIdentity(null); + sessionStorage.removeItem(STATE_KEY); + sessionStorage.removeItem(VERIFIER_KEY); + if (config.mode === "cognito_rbac" && config.client_id && config.domain) { + const query = new URLSearchParams({ + client_id: config.client_id, + logout_uri: `${window.location.origin}/`, + }); + window.location.assign(`${config.domain}/logout?${query.toString()}`); + } + }; + + const value: AuthValue = { + loading, + authenticated: Boolean(identity), + mode: config.mode, + identity, + role: identity?.roles[0] || null, + error, + login, + logout, + }; + + return {children}; +} diff --git a/frontend/src/auth/authState.ts b/frontend/src/auth/authState.ts new file mode 100644 index 0000000..d2a76af --- /dev/null +++ b/frontend/src/auth/authState.ts @@ -0,0 +1,30 @@ +import { createContext, useContext } from "react"; + + +export type AppRole = "customer" | "operator" | "admin"; + +export type Identity = { + sub: string; + username?: string; + roles: AppRole[]; + email?: string; +}; + +export type AuthValue = { + loading: boolean; + authenticated: boolean; + mode: string; + identity: Identity | null; + role: AppRole | null; + error: string | null; + login: () => Promise; + logout: () => void; +}; + +export const AuthContext = createContext(null); + +export function useAuth(): AuthValue { + const value = useContext(AuthContext); + if (!value) throw new Error("useAuth must be used inside AuthProvider"); + return value; +} diff --git a/frontend/src/components/DonorDirectoryPage.tsx b/frontend/src/components/DonorDirectoryPage.tsx index 932b733..6bb4a81 100644 --- a/frontend/src/components/DonorDirectoryPage.tsx +++ b/frontend/src/components/DonorDirectoryPage.tsx @@ -30,7 +30,8 @@ import { type DonorDirectoryState, type GrantScope, } from "../lib/grantScope"; -import { mutationHeaders } from "../lib/http"; +import { apiFetch, mutationHeaders } from "../lib/http"; +import { pollDurableJob } from "../lib/jobs"; type ProfileLink = | { status: "none" } @@ -187,11 +188,9 @@ type EnrichmentDialogState = { }; type EnrichmentRun = { - status: "running" | "success" | "failed"; + status: "queued" | "running" | "success" | "failed"; current: number; total: number; - progress?: number; - startedAt?: number; message: string; donorKeys: string[]; profiles: Array<{ @@ -212,6 +211,7 @@ export type HeaderContextState = { interface Props { apiBase: string; online: boolean; + canOperate: boolean; selectedSources: string[]; onHeaderStateChange: (state: HeaderContextState) => void; onBackToLandscape: (scope: GrantScope) => void; @@ -305,10 +305,8 @@ function charityCommissionNumber(item: DonorResult): number | null { return Number.isSafeInteger(number) && number > 0 ? number : null; } -const LOCAL_PROFILE_LINK_DURATION_MS = 7_000; - function enrichmentIsActive(run: EnrichmentRun | null): boolean { - return run?.status === "running"; + return run?.status === "queued" || run?.status === "running"; } function profileStatusLabel(profile: ProfileLink): string { @@ -347,6 +345,7 @@ function evidenceOrganizationName(evidence: EvidenceLink): string { export default function DonorDirectoryPage({ apiBase, online, + canOperate, selectedSources, onHeaderStateChange, onBackToLandscape, @@ -385,19 +384,12 @@ export default function DonorDirectoryPage({ const [enrichmentDialog, setEnrichmentDialog] = useState(null); const [enrichmentRun, setEnrichmentRun] = useState(null); const [enrichmentError, setEnrichmentError] = useState(null); - const [sourceFunderRevision, setSourceFunderRevision] = useState(0); const requestVersion = useRef(0); const detailVersion = useRef(0); const listScrollPosition = useRef(0); const drawerRef = useRef(null); const detailRef = useRef(null); const selectedRowRef = useRef(null); - const selectedSourceFunder = detail && detail.funder.source_funder_key === selectedKey - ? detail.funder - : result?.items.find(item => item.source_funder_key === selectedKey); - const activeSourceFunderKey = selectedKey; - const activeSourceFunderName = selectedSourceFunder?.display_name; - const filterCount = grantScopeFilterCount(scope) + Number(Boolean(directory.search)) + Number(directory.status !== "all") @@ -522,8 +514,8 @@ export default function DonorDirectoryPage({ params.set("page_size", "25"); setLoading(true); setError(null); - fetch(`${apiBase}/api/charities/grants/funders?${params.toString()}`, { - credentials: "include", + apiFetch(`${apiBase}/api/charities/grants/funders?${params.toString()}`, { + credentials: "omit", signal: controller.signal, }) .then(async response => { @@ -543,7 +535,7 @@ export default function DonorDirectoryPage({ if (version === requestVersion.current) setLoading(false); }); return () => controller.abort(); - }, [apiBase, directory.page, directory.search, directory.sort, directory.status, online, scope, sourceFunderRevision]); + }, [apiBase, directory.page, directory.search, directory.sort, directory.status, online, scope]); const fetchDetail = useCallback((key: string, full: boolean) => { if (!online || !scope.beneficiaryCountry) return; @@ -553,8 +545,8 @@ export default function DonorDirectoryPage({ params.set("detail_level", full ? "full" : "summary"); setDetailLoading(true); setDetailError(null); - fetch(`${apiBase}/api/charities/grants/funders/${encodeURIComponent(key)}?${params.toString()}`, { - credentials: "include", + apiFetch(`${apiBase}/api/charities/grants/funders/${encodeURIComponent(key)}?${params.toString()}`, { + credentials: "omit", signal: controller.signal, }) .then(async response => { @@ -578,102 +570,10 @@ export default function DonorDirectoryPage({ return () => controller.abort(); }, [apiBase, online, scope]); - useEffect(() => { - window.dispatchEvent(new CustomEvent("active-source-funder-change", { - detail: activeSourceFunderKey ? { - sourceFunderKey: activeSourceFunderKey, - displayName: activeSourceFunderName || "Source funder", - } : null, - })); - }, [activeSourceFunderKey, activeSourceFunderName]); - - useEffect(() => () => { - window.dispatchEvent(new CustomEvent("active-source-funder-change", { detail: null })); - }, []); - - useEffect(() => { - const refreshResetSourceFunder = (event: Event) => { - const sourceFunderKey = String( - (event as CustomEvent<{ sourceFunderKey?: string }>).detail?.sourceFunderKey || "", - ); - if (!sourceFunderKey) return; - detailVersion.current += 1; - setDetailLoading(false); - if (selectedKey === sourceFunderKey) { - setDetail(null); - setActivityLoaded(false); - } - setSourceFunderRevision(current => current + 1); - }; - window.addEventListener("source-funder-reset-to-observed", refreshResetSourceFunder); - return () => window.removeEventListener("source-funder-reset-to-observed", refreshResetSourceFunder); - }, [selectedKey]); - - const applyConfirmedProfileLinks = useCallback((profiles: EnrichmentRun["profiles"]) => { - // A confirmed GB-CHC identifier is the one safe immediate link we can - // render without making the user wait for a complete dashboard-index - // rebuild. Name-selected registry candidates remain deliberately - // unlinked until an explicit backend relationship exists. - const confirmed = new Map( - profiles - .filter(profile => profile.resolution === "confirmed_identifier") - .map(profile => [profile.donorKey, profile]), - ); - if (!confirmed.size) return; - - setDetail(current => { - if (!current) return current; - const profile = confirmed.get(current.funder.source_funder_key); - if (!profile) return current; - return { - ...current, - funder: { - ...current.funder, - profile_link: { - status: "single", - profile_id: profile.profileId, - profile_name: profile.profileName, - }, - }, - }; - }); - - setResult(current => { - if (!current) return current; - let newlyLinked = 0; - const items = current.items.map(item => { - const profile = confirmed.get(item.source_funder_key); - if (!profile) return item; - if (item.profile_link.status !== "single") newlyLinked += 1; - return { - ...item, - profile_link: { - status: "single" as const, - profile_id: profile.profileId, - profile_name: profile.profileName, - }, - }; - }); - if (!newlyLinked) return { ...current, items }; - return { - ...current, - items, - summary: { - ...current.summary, - status_counts: { - ...current.summary.status_counts, - linked: current.summary.status_counts.linked + newlyLinked, - observed_only: Math.max(0, current.summary.status_counts.observed_only - newlyLinked), - }, - }, - }; - }); - }, []); - useEffect(() => { if (!selectedKey) return; return fetchDetail(selectedKey, isFavoriteDetail); - }, [fetchDetail, isFavoriteDetail, selectedKey, sourceFunderRevision]); + }, [fetchDetail, isFavoriteDetail, selectedKey]); useEffect(() => { if (!selectedKey) return; @@ -737,7 +637,7 @@ export default function DonorDirectoryPage({ }; const beginEnrichmentSelection = async (donor: DonorResult) => { - if (!online || enrichmentIsActive(enrichmentRun)) return; + if (!canOperate || !online || enrichmentIsActive(enrichmentRun)) return; const existing = enrichmentQueue.find(item => item.donorKey === donor.source_funder_key); if (existing) { removeFromEnrichmentQueue(donor.source_funder_key); @@ -752,8 +652,8 @@ export default function DonorDirectoryPage({ setEnrichmentDialog({ donor, candidates: [], loading: true, error: null }); try { const params = new URLSearchParams({ query: donor.display_name, limit: "5", sort: "name" }); - const response = await fetch(`${apiBase}/api/charities/directory/organizations?${params.toString()}`, { - credentials: "include", + const response = await apiFetch(`${apiBase}/api/charities/directory/organizations?${params.toString()}`, { + credentials: "omit", }); const body = await response.json(); if (!response.ok) throw new Error(body.detail || "Could not search the cached Charity Commission register."); @@ -769,7 +669,7 @@ export default function DonorDirectoryPage({ }; const startEnrichmentRun = async () => { - if (!online || !enrichmentQueue.length || enrichmentIsActive(enrichmentRun)) return; + if (!canOperate || !online || !enrichmentQueue.length || enrichmentIsActive(enrichmentRun)) return; setEnrichmentError(null); const queue = [...enrichmentQueue]; const total = queue.length; @@ -780,45 +680,51 @@ export default function DonorDirectoryPage({ profileName: item.officialName, resolution: item.resolution, })); - const startedAt = Date.now(); setEnrichmentRun({ - status: "running", + status: "queued", current: 0, total, - progress: 6, - startedAt, donorKeys, profiles, - message: "Confirming the official record", + message: "Queued for worker processing", }); try { - const [response] = await Promise.all([ - fetch(`${apiBase}/api/charities/grants/funders/enrich`, { - method: "POST", - credentials: "include", - headers: mutationHeaders("enrich source funders", true), - body: JSON.stringify({ reg_numbers: queue.map(item => item.charityNumber) }), + const response = await apiFetch(`${apiBase}/api/charities/grants/funders/enrich`, { + method: "POST", + credentials: "omit", + headers: mutationHeaders("enrich source funders", true), + body: JSON.stringify({ + reg_numbers: queue.map(item => item.charityNumber), + targets: queue.map(item => ({ + source_funder_key: item.donorKey, + profile_id: item.charityNumber, + })), }), - new Promise(resolve => window.setTimeout(resolve, LOCAL_PROFILE_LINK_DURATION_MS)), - ]); + }); const body = await response.json(); if (!response.ok) throw new Error(body.detail || "Could not start organization enrichment."); - // Confirmed registry records are linked locally in one short transaction. - // Do not enter the global pipeline poll here: a source refresh must never - // delay the user from opening the newly linked profile. - if (body.status === "success") { - setEnrichmentRun({ - status: "success", - current: total, - total, - progress: 100, - donorKeys, - profiles, - message: `${total} organization${total === 1 ? "" : "s"} added to Organization Research`, - }); - setEnrichmentQueue([]); - applyConfirmedProfileLinks(profiles); + const completed = await pollDurableJob(apiBase, body.job_id, job => { + if (job.status !== "queued" && job.status !== "running") return; + setEnrichmentRun(current => current ? { + ...current, + status: job.status === "queued" ? "queued" : "running", + message: job.status === "queued" ? "Queued for worker processing" : "Linking the confirmed profile", + } : current); + }); + if (completed.status !== "succeeded") { + throw new Error(completed.failure_reason || completed.error_message || "Organization enrichment failed."); } + setEnrichmentRun({ + status: "success", + current: total, + total, + donorKeys, + profiles, + message: `${total} organization${total === 1 ? "" : "s"} added to Organization Research`, + }); + setEnrichmentQueue([]); + setDirectory(current => ({ ...current })); + if (selectedKey) fetchDetail(selectedKey, activityLoaded); } catch (reason) { const message = (reason as Error).message || "Could not start organization enrichment."; setEnrichmentRun({ status: "failed", current: 0, total, donorKeys, profiles, message: "Enrichment could not start", error: message }); @@ -826,26 +732,6 @@ export default function DonorDirectoryPage({ } }; - useEffect(() => { - if (enrichmentRun?.status !== "running") return; - const startedAt = enrichmentRun.startedAt || Date.now(); - const updateProgress = () => { - const elapsed = Math.max(0, Date.now() - startedAt); - const progress = Math.min(94, Math.round(6 + (elapsed / LOCAL_PROFILE_LINK_DURATION_MS) * 88)); - const message = progress < 35 - ? "Confirming the official record" - : progress < 72 - ? "Creating the Organization Research profile" - : "Linking already observed grants"; - setEnrichmentRun(current => current?.status === "running" && current.startedAt === startedAt - ? { ...current, progress, message } - : current); - }; - updateProgress(); - const timer = window.setInterval(updateProgress, 180); - return () => window.clearInterval(timer); - }, [enrichmentRun?.startedAt, enrichmentRun?.status]); - const applyFilters = () => { if (draft.dateFrom && draft.dateTo && draft.dateFrom > draft.dateTo) { setError("The period start cannot be after the period end."); @@ -1008,7 +894,7 @@ export default function DonorDirectoryPage({ {requestIsFavorite ? "Saved request" : "Save request"} )} - {!isFavoritePresentation && enrichmentQueue.length > 0 && ( + {canOperate && !isFavoritePresentation && enrichmentQueue.length > 0 && ( } - {!isFavoritePresentation && queuedEnrichment && } {registryEnrichment &&
{registryEnrichment.message} - {registryEnrichment.progress}% + {registryEnrichment.status === "queued" ? "Queued" : registryEnrichment.status === "running" ? "Running" : registryEnrichment.status === "success" ? "Complete" : "Failed"}
-
+ {(registryEnrichment.status === "queued" || registryEnrichment.status === "running") && } {registryEnrichment.error && {registryEnrichment.error}}
} } diff --git a/frontend/src/components/UserManagementPage.tsx b/frontend/src/components/UserManagementPage.tsx new file mode 100644 index 0000000..493e514 --- /dev/null +++ b/frontend/src/components/UserManagementPage.tsx @@ -0,0 +1,116 @@ +import { useCallback, useEffect, useState } from "react"; +import { LoaderCircle, UserPlus } from "lucide-react"; + +import { apiFetch, mutationHeaders } from "../lib/http"; + + +type AppRole = "customer" | "operator" | "admin"; + +type ManagedUser = { + id: string; + email: string | null; + enabled: boolean; + status: string; + role: AppRole | null; +}; + +type UserPage = { + users: ManagedUser[]; + next_token: string | null; +}; + + +export default function UserManagementPage({ apiBase }: { apiBase: string }) { + const [users, setUsers] = useState([]); + const [nextToken, setNextToken] = useState(null); + const [loading, setLoading] = useState(true); + const [pending, setPending] = useState(null); + const [error, setError] = useState(null); + const [email, setEmail] = useState(""); + const [role, setRole] = useState("customer"); + + const load = useCallback(async (token?: string) => { + setLoading(true); + setError(null); + try { + const query = new URLSearchParams({ page_size: "50" }); + if (token) query.set("next_token", token); + const response = await apiFetch(`${apiBase}/api/admin/users?${query.toString()}`, { credentials: "omit" }); + if (!response.ok) throw new Error(`User list request failed (${response.status}).`); + const page = await response.json() as UserPage; + setUsers(current => token ? [...current, ...page.users] : page.users); + setNextToken(page.next_token); + } catch (caught) { + setError((caught as Error).message || "User management is unavailable."); + } finally { + setLoading(false); + } + }, [apiBase]); + + useEffect(() => { void load(); }, [load]); + + const parseFailure = async (response: Response): Promise => { + const body = await response.json().catch(() => ({})); + return typeof body.detail === "string" ? body.detail : `User operation failed (${response.status}).`; + }; + + const invite = async () => { + setPending("invite"); + setError(null); + try { + const response = await apiFetch(`${apiBase}/api/admin/users`, { + method: "POST", + credentials: "omit", + headers: mutationHeaders("invite Cognito user", true), + body: JSON.stringify({ email, role }), + }); + if (!response.ok) throw new Error(await parseFailure(response)); + setEmail(""); + await load(); + } catch (caught) { + setError((caught as Error).message || "User invitation failed."); + } finally { + setPending(null); + } + }; + + const mutate = async (user: ManagedUser, action: "role" | "disable" | "enable" | "reset-password", nextRole?: AppRole) => { + setPending(`${user.id}:${action}`); + setError(null); + const suffix = action === "role" ? "role" : action; + try { + const response = await apiFetch(`${apiBase}/api/admin/users/${encodeURIComponent(user.id)}/${suffix}`, { + method: action === "role" ? "PATCH" : "POST", + credentials: "omit", + headers: mutationHeaders(`admin user ${action}`, action === "role"), + body: action === "role" ? JSON.stringify({ role: nextRole }) : undefined, + }); + if (!response.ok) throw new Error(await parseFailure(response)); + await load(); + } catch (caught) { + setError((caught as Error).message || "User operation failed."); + } finally { + setPending(null); + } + }; + + return ( +
+
+
Administration

User Management

Invite users and maintain exactly one application role per account.

+
+ {error &&
{error}
} +
+ + + +
+
+ {loading && users.length === 0 ?

Loading users…

: ( +
{users.map(user => )}
UserStatusRoleActions
{user.email || "Email unavailable"}{user.id}{user.enabled ? user.status : "DISABLED"}
+ )} + {nextToken && } +
+
+ ); +} diff --git a/frontend/src/index.css b/frontend/src/index.css index b656604..19ca5b8 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -186,43 +186,18 @@ body { border: 1px solid rgba(102, 100, 241, 0.32); } -.user-avatar-reset { - flex: 0 0 auto; - padding: 0; - cursor: pointer; - font: inherit; - line-height: 1; - transition: transform 160ms ease, background-color 160ms ease, border-color 160ms ease; -} - -.user-avatar-reset:hover { - transform: rotate(-8deg) scale(1.05); - background-color: rgba(102, 100, 241, 0.18); - border-color: var(--nl-unicorn); -} - -.user-avatar-reset:focus-visible { - outline: 3px solid rgba(102, 100, 241, 0.42); - outline-offset: 3px; -} - -.user-avatar-reset:disabled { - cursor: wait; - opacity: .72; -} - -.user-avatar-reset-spinner { - animation: profile-loading-spin .9s linear infinite; -} - .user-info { display: flex; + min-width: 0; flex-direction: column; + align-items: flex-start; } .user-name { + max-width: 100%; font-size: 14px; font-weight: 600; + overflow-wrap: anywhere; } .user-role { @@ -2548,6 +2523,7 @@ body { background: var(--nl-unicorn); transition: width .35s ease; } +.registry-enrichment-progress-track.is-indeterminate i { width: 38%; animation: donor-link-indeterminate 1.2s ease-in-out infinite; } .registry-enrichment.is-success { border-color: rgba(22, 139, 14, .32); background: rgba(200, 255, 192, .2); } .registry-enrichment.is-success .registry-enrichment-progress-track i { background: var(--semantic-success); } @@ -3146,7 +3122,7 @@ body { .profile-table-loading { display: inline-flex; align-items: center; gap: 7px; } @keyframes profile-loading-spin { to { transform: rotate(360deg); } } @media (max-width: 720px) { .profile-background-loading { grid-template-columns: auto minmax(0, 1fr); } .profile-background-loading-tasks { grid-column: 1 / -1; justify-content: flex-start; } } -@media (prefers-reduced-motion: reduce) { .profile-background-loading-spinner, .profile-section-loading svg, .profile-table-loading svg, .user-avatar-reset { animation: none; transition: none; } .user-avatar-reset:hover { transform: none; } } +@media (prefers-reduced-motion: reduce) { .profile-background-loading-spinner, .profile-section-loading svg, .profile-table-loading svg { animation: none; transition: none; } } .profile-score-breakdown { margin-top: 4px; } .profile-score-breakdown summary { color: var(--nl-unicorn-active); font-size: 12px; font-weight: 750; cursor: pointer; } .profile-score-status { color: var(--nl-unicorn-active); border-color: rgba(102, 100, 241, .3); background: var(--nl-unicorn-glow); font-weight: 700; } @@ -3659,6 +3635,8 @@ body { .donor-profile-link-steps .is-current { border-color: rgba(102,100,241,.34); color: var(--nl-unicorn-active); background: rgba(198,198,255,.3); } .donor-profile-link-progress { height: 6px; overflow: hidden; margin-top: 4px; border-radius: 999px; background: rgba(102,100,241,.12); } .donor-profile-link-progress i { display: block; height: 100%; border-radius: inherit; background: var(--nl-unicorn); transition: width .18s ease-out; } +.donor-profile-link-progress.is-indeterminate i { width: 38%; animation: donor-link-indeterminate 1.2s ease-in-out infinite; } +@keyframes donor-link-indeterminate { from { transform: translateX(-110%); } to { transform: translateX(285%); } } .donor-profile-link-progress-label { align-self: flex-end; color: var(--nl-unicorn-active); font-size: 9px; font-weight: 800; } .donor-enrichment-progress { display: flex; align-items: flex-start; gap: 8px; margin-top: 10px; border: 1px solid rgba(102,100,241,.22); border-radius: var(--radius-sm); padding: 9px; background: var(--nl-unicorn-glow); } .donor-enrichment-progress.success { border-color: rgba(50,223,25,.28); background: rgba(200,250,192,.28); } @@ -3918,3 +3896,79 @@ textarea { .landscape-metrics { grid-template-columns: 1fr !important; } .header-title { max-width: 104px; } } + +.auth-shell { + min-height: 100vh; + display: grid; + place-items: center; + padding: 24px; + background: var(--bg-page); +} + +.auth-card { + width: min(460px, 100%); + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 18px; + padding: 32px; +} + +.auth-card img { width: 118px; height: auto; } +.auth-card h1 { font-size: 28px; line-height: 1.15; } +.auth-card p { color: var(--text-secondary); } +.sidebar-auth-action { + display: inline-flex; + min-height: 32px; + align-items: center; + gap: 5px; + margin-top: 5px; + border: 1px solid var(--border-glass); + border-radius: var(--radius-sm); + padding: 4px 7px; + color: var(--text-secondary); + background: #fff; + font: inherit; + font-size: 11px; + font-weight: 700; + white-space: nowrap; + cursor: pointer; +} +.sidebar-auth-action:hover { border-color: var(--nl-unicorn); color: var(--nl-unicorn); } +.sidebar-auth-action:focus-visible { outline: 3px solid var(--border-glass-focus); outline-offset: 2px; } +.sidebar-auth-error { padding: 0 8px; color: var(--semantic-error); font-size: 11px; line-height: 1.35; } +.sidebar-collapsed .user-avatar { display: none; } +.sidebar-collapsed .sidebar-auth-action { margin-left: 0; } +.sidebar-collapsed .sidebar-auth-action span { display: none; } + +.scraper-source-status { + display: flex; + justify-content: space-between; + gap: 16px; + align-items: center; + padding-top: 12px; + border-top: 1px solid var(--border-glass); +} + +.scraper-source-status > span:first-child { display: flex; flex-direction: column; gap: 3px; } +.scraper-source-status small { color: var(--text-secondary); } + +.user-invite-card { + display: grid; + grid-template-columns: minmax(220px, 1fr) minmax(160px, 220px) auto; + gap: 16px; + align-items: end; +} + +.user-table-card { display: flex; flex-direction: column; gap: 16px; } +.user-table-scroll { overflow-x: auto; } +.user-table { width: 100%; border-collapse: collapse; } +.user-table th, .user-table td { text-align: left; padding: 12px; border-bottom: 1px solid var(--border-glass); vertical-align: middle; } +.user-table td:first-child { display: flex; flex-direction: column; gap: 3px; } +.user-table small { color: var(--text-muted); font-family: var(--font-mono); } +.user-row-actions { display: flex; flex-wrap: wrap; gap: 8px; } + +@media (max-width: 760px) { + .user-invite-card { grid-template-columns: 1fr; } + .scraper-source-status { align-items: flex-start; flex-direction: column; } +} diff --git a/frontend/src/lib/avatar.ts b/frontend/src/lib/avatar.ts new file mode 100644 index 0000000..42e9f25 --- /dev/null +++ b/frontend/src/lib/avatar.ts @@ -0,0 +1,14 @@ +export function avatarInitials(identity: unknown, authenticated: boolean): string { + if (!authenticated) return "NG"; + const raw = typeof identity === "string" ? identity.trim() : ""; + const localPart = raw.split("@", 1)[0] || ""; + const components = localPart + .split(/[._-]+/) + .map(component => component.replace(/[^a-z0-9]/gi, "")) + .filter(Boolean); + if (components.length >= 2) { + return `${components[0][0]}${components[1][0]}`.toUpperCase(); + } + const usable = (components[0] || localPart).replace(/[^a-z0-9]/gi, ""); + return usable.slice(0, 2).toUpperCase() || "AU"; +} diff --git a/frontend/src/lib/http.ts b/frontend/src/lib/http.ts index 62d9275..056a6c3 100644 --- a/frontend/src/lib/http.ts +++ b/frontend/src/lib/http.ts @@ -1,3 +1,16 @@ +let accessTokenProvider: (() => string | null) | null = null; + +export function setAccessTokenProvider(provider: (() => string | null) | null): void { + accessTokenProvider = provider; +} + +export function apiFetch(input: RequestInfo | URL, init: RequestInit = {}): Promise { + const headers = new Headers(init.headers); + const token = accessTokenProvider?.(); + if (token) headers.set("Authorization", `Bearer ${token}`); + return fetch(input, { ...init, headers }); +} + export function mutationHeaders(reason: string, json = false): Record { const key = typeof globalThis.crypto?.randomUUID === "function" ? globalThis.crypto.randomUUID() diff --git a/frontend/src/lib/jobs.ts b/frontend/src/lib/jobs.ts new file mode 100644 index 0000000..6c63ad1 --- /dev/null +++ b/frontend/src/lib/jobs.ts @@ -0,0 +1,42 @@ +import { apiFetch } from "./http.js"; + +export type DurableJob = { + job_id: string; + status: "queued" | "running" | "succeeded" | "failed" | "cancelled" | "timed_out" | "dead_lettered"; + result?: Record | null; + error_message?: string | null; + failure_reason?: string | null; +}; + +const terminal = new Set(["succeeded", "failed", "cancelled", "timed_out", "dead_lettered"]); + +export async function pollDurableJob( + apiBase: string, + jobId: string, + onUpdate: (job: DurableJob) => void, + signal?: AbortSignal, + intervalMs = 1_000, +): Promise { + for (;;) { + const response = await apiFetch( + `${apiBase}/api/admin/pipeline/jobs/${encodeURIComponent(jobId)}`, + { credentials: "omit", signal }, + ); + const body = await response.json(); + if (!response.ok) throw new Error(body.detail || `Job status request failed (${response.status}).`); + const job = body as DurableJob; + onUpdate(job); + if (terminal.has(job.status)) return job; + await new Promise((resolve, reject) => { + const onAbort = () => { + window.clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }; + const timer = window.setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, intervalMs); + signal?.addEventListener("abort", onAbort, { once: true }); + }); + } +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index bef5202..6067a5f 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -2,9 +2,12 @@ import { StrictMode } from 'react' import { createRoot } from 'react-dom/client' import './index.css' import App from './App.tsx' +import { AuthProvider } from './auth/AuthContext.tsx' createRoot(document.getElementById('root')!).render( - + + + , ) diff --git a/frontend/tests/authContracts.test.ts b/frontend/tests/authContracts.test.ts new file mode 100644 index 0000000..98e31c9 --- /dev/null +++ b/frontend/tests/authContracts.test.ts @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; + +import { apiFetch, setAccessTokenProvider } from "../src/lib/http.js"; + + +const source = (relativePath: string) => readFileSync(join(process.cwd(), relativePath), "utf8"); + +test("Managed Login uses authorization code plus PKCE and removes callback data", () => { + const auth = source("src/auth/AuthContext.tsx"); + + assert.match(auth, /response_type:\s*"code"/); + assert.match(auth, /code_challenge_method:\s*"S256"/); + assert.match(auth, /code_verifier:\s*verifier/); + assert.match(auth, /returnedState\s*!==\s*expectedState/); + assert.match(auth, /history\.replaceState\(\{\},\s*"",\s*"\/"\)/); + assert.doesNotMatch(auth, /localStorage/); + assert.match(auth, /sessionStorage\.setItem\(TOKEN_KEY/); +}); + +test("authenticated API requests use bearer tokens without browser credentials", async () => { + const originalFetch = globalThis.fetch; + let captured: RequestInit | undefined; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + captured = init; + return new Response(null, { status: 204 }); + }) as typeof fetch; + + try { + setAccessTokenProvider(() => "verified-access-token"); + await apiFetch("/api/example", { credentials: "omit" }); + const headers = new Headers(captured?.headers); + assert.equal(headers.get("Authorization"), "Bearer verified-access-token"); + assert.equal(captured?.credentials, "omit"); + } finally { + setAccessTokenProvider(null); + globalThis.fetch = originalFetch; + } +}); + +test("role-gated UI keeps operational and user administration controls restricted", () => { + const app = source("src/App.tsx"); + const donor = source("src/components/DonorDirectoryPage.tsx"); + const registry = source("src/components/RegistryDirectory.tsx"); + const users = source("src/components/UserManagementPage.tsx"); + const allFrontend = `${app}\n${donor}\n${registry}\n${users}`; + + assert.match(app, /const canOperate = auth\.role === "operator" \|\| auth\.role === "admin"/); + assert.match(app, /const isAdmin = auth\.role === "admin"/); + assert.match(app, /isAdmin && .*User Management/s); + assert.match(app, /canOperate && { + const app = source("src/App.tsx"); + const donor = source("src/components/DonorDirectoryPage.tsx"); + const styles = source("src/index.css"); + + assert.match(app, /"Netlight Guest"/); + assert.match(app, /"Not signed in"/); + assert.match(app, /onClick=\{\(\) => void auth\.login\(\)\}/); + assert.match(app, />Sign in<\/span>/); + assert.match(app, />Sign out<\/span>/); + assert.match(app, /auth\.identity\?\.email \|\| auth\.identity\?\.username/); + assert.match(app, /avatarInitials\(authenticatedIdentity, auth\.authenticated\)/); + assert.match(app, /auth\.authenticated \? auth\.role : "Not signed in"/); + assert.match(app, /onClick=\{auth\.logout\}/); + assert.doesNotMatch(app, />NLAdministrator { + assert.equal(avatarInitials("manuel.grabmayer@tum.de", true), "MG"); + assert.equal(avatarInitials("john.smith@example.com", true), "JS"); + assert.equal(avatarInitials("anna-maria@example.com", true), "AM"); + assert.equal(avatarInitials("max_mueller@example.com", true), "MM"); + assert.equal(avatarInitials("manuel@tum.de", true), "MA"); + assert.equal(avatarInitials("x@example.com", true), "X"); +}); + +test("avatar initials safely fall back for malformed and guest identities", () => { + assert.equal(avatarInitials("!!!@example.com", true), "AU"); + assert.equal(avatarInitials(null, true), "AU"); + assert.equal(avatarInitials("manuel.grabmayer@tum.de", false), "NG"); +}); diff --git a/frontend/tests/workerUiContracts.test.ts b/frontend/tests/workerUiContracts.test.ts new file mode 100644 index 0000000..bb72094 --- /dev/null +++ b/frontend/tests/workerUiContracts.test.ts @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; + +const source = (relativePath: string) => readFileSync(join(process.cwd(), relativePath), "utf8"); + +test("organisation enrichment polls durable backend state and refetches persisted data", () => { + const donor = source("src/components/DonorDirectoryPage.tsx"); + const registry = source("src/components/RegistryDirectory.tsx"); + const jobs = source("src/lib/jobs.ts"); + + assert.match(donor, /pollDurableJob\(apiBase, body\.job_id/); + assert.match(donor, /fetchDetail\(selectedKey, activityLoaded\)/); + assert.match(registry, /pollDurableJob\(apiBase, payload\.job_id/); + assert.match(registry, /setDetailRefreshRevision\(current => current \+ 1\)/); + assert.match(jobs, /job\.status/); + assert.match(jobs, /terminal\.has\(job\.status\)/); +}); + +test("queued and running remain distinct and fabricated numeric progress is absent", () => { + const donor = source("src/components/DonorDirectoryPage.tsx"); + const registry = source("src/components/RegistryDirectory.tsx"); + const app = source("src/App.tsx"); + const combined = `${donor}\n${registry}\n${app}`; + + assert.match(donor, /status: job\.status === "queued" \? "queued" : "running"/); + assert.match(registry, /status: job\.status === "queued" \? "queued" : "running"/); + assert.match(combined, /is-indeterminate/); + assert.doesNotMatch(combined, /Math\.min\(94/); + assert.doesNotMatch(combined, /progress:\s*94/); +}); + +test("failed durable jobs expose a safe terminal error instead of running forever", () => { + const donor = source("src/components/DonorDirectoryPage.tsx"); + const registry = source("src/components/RegistryDirectory.tsx"); + assert.match(donor, /completed\.failure_reason \|\| completed\.error_message/); + assert.match(registry, /completed\.failure_reason \|\| completed\.error_message/); + assert.match(donor, /status: "failed"/); + assert.match(registry, /status: "failed"/); +}); diff --git a/infra/cloudformation/db-access-prerequisite.yaml b/infra/cloudformation/db-access-prerequisite.yaml new file mode 100644 index 0000000..ed38627 --- /dev/null +++ b/infra/cloudformation/db-access-prerequisite.yaml @@ -0,0 +1,210 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: >- + Isolated database-access prerequisite for the Foundation Intelligence demo. + Creates a generated writer secret and a one-off ARM64 release task definition; + it does not run the task, update the application service, or modify RDS. + +Parameters: + BaseStackName: + Type: String + Default: foundation-intelligence-demo-e06ab1ea-v2 + AllowedPattern: '^[A-Za-z][A-Za-z0-9-]{0,127}$' + BackendImageUri: + Type: String + Description: Immutable ARM64 backend image containing migration.database_access. + AllowedPattern: '^208337080387\.dkr\.ecr\.eu-west-1\.amazonaws\.com/foundation-intelligence-platform/backend:(?!latest$)[A-Za-z0-9._-]+(@sha256:[a-f0-9]{64})?$' + DatabaseEndpoint: + Type: String + AllowedPattern: '^[A-Za-z0-9.-]+\.rds\.amazonaws\.com$' + DatabaseName: + Type: String + Default: foundation_intelligence + AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + DatabaseReaderUser: + Type: String + Default: foundation_app + AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + DatabaseWriterUser: + Type: String + Default: foundation_app_writer + AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + DatabasePipelineUser: + Type: String + Default: foundation_pipeline_writer + AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + DatabaseMasterSecretArn: + Type: String + AllowedPattern: '^arn:aws:secretsmanager:eu-west-1:208337080387:secret:[A-Za-z0-9/_+=.@!-]+$' + DatabaseReaderSecretArn: + Type: String + AllowedPattern: '^arn:aws:secretsmanager:eu-west-1:208337080387:secret:[A-Za-z0-9/_+=.@!-]+$' + LogRetentionDays: + Type: Number + Default: 7 + AllowedValues: [7, 14, 30] + +Rules: + RequireDistinctRuntimeDatabasePrincipals: + Assertions: + - Assert: !Not + - !Equals [!Ref DatabaseReaderUser, !Ref DatabaseWriterUser] + AssertDescription: Reader and writer PostgreSQL principals must differ. + - Assert: !Not + - !Equals [!Ref DatabaseReaderUser, !Ref DatabasePipelineUser] + AssertDescription: Reader and pipeline PostgreSQL principals must differ. + - Assert: !Not + - !Equals [!Ref DatabaseWriterUser, !Ref DatabasePipelineUser] + AssertDescription: Writer and pipeline PostgreSQL principals must differ. + +Resources: + WriterDatabaseSecret: + Type: AWS::SecretsManager::Secret + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + Description: Restricted DML writer for the Foundation Intelligence runtime + GenerateSecretString: + SecretStringTemplate: !Sub '{"username":"${DatabaseWriterUser}"}' + GenerateStringKey: password + PasswordLength: 40 + ExcludePunctuation: true + Tags: + - Key: Environment + Value: demo + - Key: Purpose + Value: runtime-database-writer + + PipelineDatabaseSecret: + Type: AWS::SecretsManager::Secret + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + Description: Restricted versioned dataset publisher for the pipeline worker + GenerateSecretString: + SecretStringTemplate: !Sub '{"username":"${DatabasePipelineUser}"}' + GenerateStringKey: password + PasswordLength: 40 + ExcludePunctuation: true + Tags: + - Key: Environment + Value: demo + - Key: Purpose + Value: pipeline-dataset-publisher + + DbAccessLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub '/foundation-intelligence/demo/${BaseStackName}/db-access-prerequisite' + RetentionInDays: !Ref LogRetentionDays + + DbAccessExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: ReadOnlyDatabasePrerequisiteSecrets + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: secretsmanager:GetSecretValue + Resource: + - !Ref DatabaseMasterSecretArn + - !Ref DatabaseReaderSecretArn + - !Ref WriterDatabaseSecret + - !Ref PipelineDatabaseSecret + + DbAccessTaskRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + + DbAccessTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub '${BaseStackName}-db-access-prerequisite' + Cpu: '512' + Memory: '1024' + NetworkMode: awsvpc + RequiresCompatibilities: [FARGATE] + ExecutionRoleArn: !GetAtt DbAccessExecutionRole.Arn + TaskRoleArn: !GetAtt DbAccessTaskRole.Arn + RuntimePlatform: + CpuArchitecture: ARM64 + OperatingSystemFamily: LINUX + ContainerDefinitions: + - Name: provision-database-access + Image: !Ref BackendImageUri + Essential: true + User: '10001:10001' + EntryPoint: [python, -m] + Command: [migration.database_access] + Environment: + - Name: APP_ENV + Value: demo + - Name: DATABASE_HOST + Value: !Ref DatabaseEndpoint + - Name: DATABASE_PORT + Value: '5432' + - Name: DATABASE_NAME + Value: !Ref DatabaseName + - Name: DATABASE_READER_USER + Value: !Ref DatabaseReaderUser + - Name: DATABASE_WRITER_USER + Value: !Ref DatabaseWriterUser + - Name: DATABASE_PIPELINE_USER + Value: !Ref DatabasePipelineUser + - Name: DATABASE_SSL_MODE + Value: require + Secrets: + - Name: DATABASE_ADMIN_USER + ValueFrom: !Sub '${DatabaseMasterSecretArn}:username::' + - Name: DATABASE_ADMIN_PASSWORD + ValueFrom: !Sub '${DatabaseMasterSecretArn}:password::' + - Name: DATABASE_READER_PASSWORD + ValueFrom: !Sub '${DatabaseReaderSecretArn}:password::' + - Name: DATABASE_WRITER_PASSWORD + ValueFrom: !Sub '${WriterDatabaseSecret}:password::' + - Name: DATABASE_PIPELINE_PASSWORD + ValueFrom: !Sub '${PipelineDatabaseSecret}:password::' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref DbAccessLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: prerequisite + Tags: + - Key: Environment + Value: demo + - Key: Purpose + Value: database-access-prerequisite + +Outputs: + WriterDatabaseSecretArn: + Description: Pass to ApplicationDatabaseWriterSecretArn only after task verification. + Value: !Ref WriterDatabaseSecret + WriterDatabaseUser: + Value: !Ref DatabaseWriterUser + PipelineDatabaseSecretArn: + Description: Pass to the application stack only for the worker container. + Value: !Ref PipelineDatabaseSecret + PipelineDatabaseUser: + Value: !Ref DatabasePipelineUser + DbAccessTaskDefinitionArn: + Value: !Ref DbAccessTaskDefinition + DbAccessLogGroupName: + Value: !Ref DbAccessLogGroup diff --git a/infra/cloudformation/demo.yaml b/infra/cloudformation/demo.yaml index 5859268..7c1748f 100644 --- a/infra/cloudformation/demo.yaml +++ b/infra/cloudformation/demo.yaml @@ -1,8 +1,9 @@ AWSTemplateFormatVersion: '2010-09-09' Description: >- - Cost-conscious, non-production Foundation Intelligence public read-only demo. - HTTP is temporary and not production-ready. DesiredCount defaults to zero and - no data, image or stack operation is performed by this template file itself. + Cost-conscious, non-production Foundation Intelligence customer pilot. + Browser TLS terminates at CloudFront while the CloudFront-to-ALB and ALB-to-ECS + hops remain HTTP-only accepted temporary residual risks. DesiredCount defaults + to zero and no data, image or stack operation is performed by this template. Parameters: VpcId: @@ -31,6 +32,11 @@ Parameters: Type: String Description: Immutable ARM64 backend ECR URI, preferably pinned by digest. AllowedPattern: '^208337080387\.dkr\.ecr\.eu-west-1\.amazonaws\.com/foundation-intelligence-platform/backend:(?!latest$)[A-Za-z0-9._-]+(@sha256:[a-f0-9]{64})?$' + MigrationImageUri: + Type: String + Default: '' + Description: Optional backend image used to apply schema before enabling its runtime. + AllowedPattern: '^$|^208337080387\.dkr\.ecr\.eu-west-1\.amazonaws\.com/foundation-intelligence-platform/backend:(?!latest$)[A-Za-z0-9._-]+(@sha256:[a-f0-9]{64})?$' FrontendImageUri: Type: String Description: Immutable ARM64 frontend-runtime-ecs ECR URI, preferably pinned by digest. @@ -56,6 +62,42 @@ Parameters: Type: String Default: foundation_app AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + Description: SELECT-only runtime reader provisioned by the release path. + ApplicationDatabaseWriterUser: + Type: String + Default: foundation_app_writer + AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + Description: Restricted DML writer; must differ from ApplicationDatabaseUser. + ApplicationDatabaseWriterSecretArn: + Type: String + Default: '' + AllowedPattern: '^$|^arn:aws:secretsmanager:eu-west-1:208337080387:secret:[A-Za-z0-9/_+=.@-]+$' + Description: >- + Output of the separately executed DB-access prerequisite stack. Keep empty + until the writer role and grants have been provisioned and verified. + ApplicationDatabasePipelineUser: + Type: String + Default: foundation_pipeline_writer + AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + Description: Dedicated versioned-dataset publisher used only by the worker. + ApplicationDatabasePipelineSecretArn: + Type: String + Default: '' + AllowedPattern: '^$|^arn:aws:secretsmanager:eu-west-1:208337080387:secret:[A-Za-z0-9/_+=.@-]+$' + Description: Output of the verified DB-access prerequisite stack. + AnthropicSecretArn: + Type: String + AllowedPattern: '^arn:aws:secretsmanager:eu-west-1:208337080387:secret:foundation-intelligence/demo/anthropic-api-key-[A-Za-z0-9]{6}$' + Description: Existing JSON secret containing the server-side Anthropic api_key. + CharityCommissionSecretArn: + Type: String + AllowedPattern: '^arn:aws:secretsmanager:eu-west-1:208337080387:secret:foundation-intelligence/demo/charity-commission-api-key-[A-Za-z0-9]{6}$' + Description: Existing JSON secret containing the worker-only Charity Commission api_key. + WorkerEnabled: + Type: String + Default: 'false' + AllowedValues: ['false', 'true'] + Description: Enable only after schema and dedicated database grants are ready. SourceObjectKey: Type: String Default: imports/charities.db @@ -76,6 +118,66 @@ Parameters: Type: String Description: Full Git commit SHA used to build the approved backend image. AllowedPattern: '^[a-f0-9]{40}$' + OriginLockdownEnabled: + Type: String + Default: 'false' + AllowedValues: ['false', 'true'] + Description: false for Deployment A; true only for the approved CloudFront-only origin lockdown. + CloudFrontOriginPrefixListId: + Type: String + Default: '' + Description: >- + Region-specific com.amazonaws.global.cloudfront.origin-facing prefix list. + It is intentionally supplied at deployment time and never hardcoded. + AllowedPattern: '^$|^pl-[a-f0-9]+$' + OriginVerificationToken: + Type: String + NoEcho: true + MinLength: 43 + MaxLength: 128 + AllowedPattern: '^[A-Za-z0-9_-]{43,128}$' + Description: >- + At least 32 bytes of cryptographic entropy encoded as base64url. NoEcho is + not protection from privileged principals who can inspect AWS configuration. + CognitoDomainPrefix: + Type: String + MinLength: 1 + MaxLength: 63 + AllowedPattern: '^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$' + Description: Globally available, project-specific Cognito managed-login prefix. + +Rules: + RequireDistinctRuntimeDatabasePrincipals: + Assertions: + - Assert: !Not + - !Equals [!Ref ApplicationDatabaseUser, !Ref ApplicationDatabaseWriterUser] + AssertDescription: Reader and writer PostgreSQL principals must differ. + - Assert: !Not + - !Equals [!Ref ApplicationDatabaseUser, !Ref ApplicationDatabasePipelineUser] + AssertDescription: Reader and pipeline PostgreSQL principals must differ. + - Assert: !Not + - !Equals [!Ref ApplicationDatabaseWriterUser, !Ref ApplicationDatabasePipelineUser] + AssertDescription: Writer and pipeline PostgreSQL principals must differ. + RequireWorkerPrerequisites: + RuleCondition: !Equals [!Ref WorkerEnabled, 'true'] + Assertions: + - Assert: !Not [!Equals [!Ref ApplicationDatabaseWriterSecretArn, '']] + AssertDescription: Runtime writer secret is required when the worker is enabled. + - Assert: !Not [!Equals [!Ref ApplicationDatabasePipelineSecretArn, '']] + AssertDescription: Pipeline publisher secret is required when the worker is enabled. + RequireCloudFrontPrefixListForLockdown: + RuleCondition: !Equals [!Ref OriginLockdownEnabled, 'true'] + Assertions: + - Assert: !Not [!Equals [!Ref CloudFrontOriginPrefixListId, '']] + AssertDescription: CloudFrontOriginPrefixListId is required when origin lockdown is enabled. + +Conditions: + OriginLockdown: !Equals [!Ref OriginLockdownEnabled, 'true'] + PublicOriginAccess: !Equals [!Ref OriginLockdownEnabled, 'false'] + RuntimeWriterConfigured: !Not [!Equals [!Ref ApplicationDatabaseWriterSecretArn, '']] + RuntimePipelineConfigured: !Not [!Equals [!Ref ApplicationDatabasePipelineSecretArn, '']] + PipelineWorkerEnabled: !Equals [!Ref WorkerEnabled, 'true'] + SeparateMigrationImage: !Not [!Equals [!Ref MigrationImageUri, '']] Resources: DatabaseSubnetA: @@ -129,16 +231,32 @@ Resources: Properties: GroupDescription: Temporary public HTTP ingress for the non-production demo VpcId: !Ref VpcId - SecurityGroupIngress: - - Description: Temporary HTTP demo only - IpProtocol: tcp - FromPort: 80 - ToPort: 80 - CidrIp: 0.0.0.0/0 Tags: - Key: Name Value: !Sub '${AWS::StackName}-alb' + PublicAlbHttpIngress: + Type: AWS::EC2::SecurityGroupIngress + Condition: PublicOriginAccess + Properties: + Description: Deployment A public HTTP origin access + GroupId: !GetAtt AlbSecurityGroup.GroupId + IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: 0.0.0.0/0 + + CloudFrontAlbHttpIngress: + Type: AWS::EC2::SecurityGroupIngress + Condition: OriginLockdown + Properties: + Description: Deployment B HTTP access from CloudFront origin-facing servers only + GroupId: !GetAtt AlbSecurityGroup.GroupId + IpProtocol: tcp + FromPort: 80 + ToPort: 80 + SourcePrefixListId: !Ref CloudFrontOriginPrefixListId + EcsSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: @@ -251,12 +369,76 @@ Resources: LogGroupName: !Sub '/foundation-intelligence/demo/${AWS::StackName}/backend' RetentionInDays: 7 + WorkerLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub '/foundation-intelligence/demo/${AWS::StackName}/worker' + RetentionInDays: 7 + MigrationLogGroup: Type: AWS::Logs::LogGroup Properties: LogGroupName: !Sub '/foundation-intelligence/demo/${AWS::StackName}/migration' RetentionInDays: 7 + CognitoUserPool: + Type: AWS::Cognito::UserPool + Properties: + UserPoolName: !Sub '${AWS::StackName}-users' + UsernameAttributes: [email] + AutoVerifiedAttributes: [email] + MfaConfiguration: 'ON' + EnabledMfas: [SOFTWARE_TOKEN_MFA] + AccountRecoverySetting: + RecoveryMechanisms: + - Name: verified_email + Priority: 1 + AdminCreateUserConfig: + AllowAdminCreateUserOnly: true + InviteMessageTemplate: + EmailSubject: Foundation Intelligence Platform invitation + EmailMessage: 'Your username is {username} and temporary password is {####}.' + Policies: + PasswordPolicy: + MinimumLength: 12 + RequireLowercase: true + RequireUppercase: true + RequireNumbers: true + RequireSymbols: true + TemporaryPasswordValidityDays: 7 + UserAttributeUpdateSettings: + AttributesRequireVerificationBeforeUpdate: [email] + VerificationMessageTemplate: + DefaultEmailOption: CONFIRM_WITH_CODE + UserPoolTier: ESSENTIALS + DeletionProtection: !If [OriginLockdown, ACTIVE, INACTIVE] + UserPoolTags: + Environment: demo + + CustomerGroup: + Type: AWS::Cognito::UserPoolGroup + Properties: + UserPoolId: !Ref CognitoUserPool + GroupName: customer + Description: Normal analytics and sanitized scraper status + Precedence: 30 + + OperatorGroup: + Type: AWS::Cognito::UserPoolGroup + Properties: + UserPoolId: !Ref CognitoUserPool + GroupName: operator + Description: Customer access plus operational scraper and pipeline controls + Precedence: 20 + + AdminGroup: + Type: AWS::Cognito::UserPoolGroup + Properties: + UserPoolId: !Ref CognitoUserPool + GroupName: admin + Description: Full administration including users, roles and critical settings + Precedence: 10 + AppDatabaseSecret: Type: AWS::SecretsManager::Secret Properties: @@ -347,7 +529,39 @@ Resources: ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy Policies: - - PolicyName: ReadDemoDatabaseSecrets + - PolicyName: ReadApplicationSecrets + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: secretsmanager:GetSecretValue + Resource: + - !Ref AppDatabaseSecret + - !Ref AnthropicSecretArn + - !Ref CharityCommissionSecretArn + - !If + - RuntimeWriterConfigured + - !Ref ApplicationDatabaseWriterSecretArn + - !Ref AWS::NoValue + - !If + - RuntimePipelineConfigured + - !Ref ApplicationDatabasePipelineSecretArn + - !Ref AWS::NoValue + + MigrationExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: ReadMigrationDatabaseSecrets PolicyDocument: Version: '2012-10-17' Statement: @@ -367,6 +581,45 @@ Resources: Principal: Service: ecs-tasks.amazonaws.com Action: sts:AssumeRole + Policies: + - PolicyName: CognitoUserAdministration + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - cognito-idp:ListUsers + - cognito-idp:ListUsersInGroup + - cognito-idp:AdminGetUser + - cognito-idp:AdminCreateUser + - cognito-idp:AdminListGroupsForUser + - cognito-idp:AdminAddUserToGroup + - cognito-idp:AdminRemoveUserFromGroup + - cognito-idp:AdminDisableUser + - cognito-idp:AdminEnableUser + - cognito-idp:AdminResetUserPassword + Resource: !GetAtt CognitoUserPool.Arn + - PolicyName: PipelineSnapshotArtifacts + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: DiscoverCurrentSnapshotFallback + Effect: Allow + Action: s3:ListBucket + Resource: !GetAtt ImportBucket.Arn + Condition: + StringEquals: + s3:prefix: pipeline-snapshots/current.db + - Sid: ReadApprovedBaselineAndCurrentSnapshot + Effect: Allow + Action: s3:GetObject + Resource: + - !Sub '${ImportBucket.Arn}/${SourceObjectKey}' + - !Sub '${ImportBucket.Arn}/pipeline-snapshots/current.db' + - Sid: PublishVersionedCurrentSnapshot + Effect: Allow + Action: s3:PutObject + Resource: !Sub '${ImportBucket.Arn}/pipeline-snapshots/current.db' MigrationTaskRole: Type: AWS::IAM::Role @@ -398,8 +651,8 @@ Resources: Type: AWS::ECS::TaskDefinition Properties: Family: !Sub '${AWS::StackName}-application' - Cpu: '512' - Memory: '2048' + Cpu: !If [PipelineWorkerEnabled, '1024', '512'] + Memory: !If [PipelineWorkerEnabled, '4096', '2048'] NetworkMode: awsvpc RequiresCompatibilities: [FARGATE] ExecutionRoleArn: !GetAtt EcsExecutionRole.Arn @@ -417,13 +670,23 @@ Resources: - Name: APP_ENV Value: demo - Name: AUTH_MODE - Value: public_readonly + Value: cognito_rbac - Name: DEV_AUTH_ENABLED Value: 'false' + - Name: COGNITO_REGION + Value: !Ref AWS::Region + - Name: COGNITO_USER_POOL_ID + Value: !Ref CognitoUserPool + - Name: COGNITO_CLIENT_ID + Value: !Ref CognitoUserPoolClient + - Name: COGNITO_DOMAIN + Value: !Sub 'https://${CognitoDomainPrefix}.auth.${AWS::Region}.amazoncognito.com' - Name: CORE_PROXY_ENABLED Value: 'false' - Name: CORS_ORIGINS Value: '' + - Name: ANTHROPIC_BASE_URL + Value: https://llm.netlight.ai/ - Name: DATA_RUNTIME_MODE Value: postgresql - Name: DATABASE_HOST @@ -434,6 +697,11 @@ Resources: Value: foundation_intelligence - Name: DATABASE_USER Value: !Ref ApplicationDatabaseUser + - !If + - RuntimeWriterConfigured + - Name: DATABASE_WRITE_USER + Value: !Ref ApplicationDatabaseWriterUser + - !Ref AWS::NoValue - Name: DATABASE_SSL_MODE Value: require - Name: DATABASE_POOL_SIZE @@ -443,6 +711,13 @@ Resources: Secrets: - Name: DATABASE_PASSWORD ValueFrom: !Sub '${AppDatabaseSecret}:password::' + - Name: ANTHROPIC_API_KEY + ValueFrom: !Sub '${AnthropicSecretArn}:api_key::' + - !If + - RuntimeWriterConfigured + - Name: DATABASE_WRITE_PASSWORD + ValueFrom: !Sub '${ApplicationDatabaseWriterSecretArn}:password::' + - !Ref AWS::NoValue HealthCheck: Command: - CMD @@ -492,6 +767,68 @@ Resources: awslogs-group: !Ref FrontendLogGroup awslogs-region: !Ref AWS::Region awslogs-stream-prefix: frontend + - !If + - PipelineWorkerEnabled + - Name: worker + Image: !Ref BackendImageUri + Essential: true + User: '10001:10001' + EntryPoint: [python, -m] + Command: [pipelines.worker] + StopTimeout: 120 + Environment: + - Name: APP_ENV + Value: demo + - Name: AUTH_MODE + Value: cognito_rbac + - Name: DATA_RUNTIME_MODE + Value: postgresql + - Name: DATABASE_HOST + Value: !GetAtt Database.Endpoint.Address + - Name: DATABASE_PORT + Value: '5432' + - Name: DATABASE_NAME + Value: foundation_intelligence + - Name: DATABASE_USER + Value: !Ref ApplicationDatabaseUser + - Name: DATABASE_WRITE_USER + Value: !Ref ApplicationDatabaseWriterUser + - Name: DATABASE_PIPELINE_USER + Value: !Ref ApplicationDatabasePipelineUser + - Name: DATABASE_SSL_MODE + Value: require + - Name: WORKER_QUEUE_NAME + Value: pipeline + - Name: WORKER_LEASE_SECONDS + Value: '90' + - Name: WORKER_IDLE_SECONDS + Value: '2' + - Name: PIPELINE_SNAPSHOT_S3_URI + Value: !Sub 's3://${ImportBucket}/pipeline-snapshots/current.db' + - Name: PIPELINE_FALLBACK_S3_URI + Value: !Sub 's3://${ImportBucket}/${SourceObjectKey}' + - Name: PIPELINE_FALLBACK_SHA256 + Value: !Ref SourceSha256 + - Name: PIPELINE_SOURCE_SCHEMA_VERSION + Value: !Ref SourceSchemaVersion + - Name: CODE_REVISION + Value: !Ref CodeRevision + Secrets: + - Name: DATABASE_PASSWORD + ValueFrom: !Sub '${AppDatabaseSecret}:password::' + - Name: DATABASE_WRITE_PASSWORD + ValueFrom: !Sub '${ApplicationDatabaseWriterSecretArn}:password::' + - Name: DATABASE_PIPELINE_PASSWORD + ValueFrom: !Sub '${ApplicationDatabasePipelineSecretArn}:password::' + - Name: CHARITY_COMMISSION_API_KEY + ValueFrom: !Sub '${CharityCommissionSecretArn}:api_key::' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref WorkerLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: worker + - !Ref AWS::NoValue Tags: - Key: Environment Value: demo @@ -506,7 +843,7 @@ Resources: Memory: '4096' NetworkMode: awsvpc RequiresCompatibilities: [FARGATE] - ExecutionRoleArn: !GetAtt EcsExecutionRole.Arn + ExecutionRoleArn: !GetAtt MigrationExecutionRole.Arn TaskRoleArn: !GetAtt MigrationTaskRole.Arn RuntimePlatform: CpuArchitecture: ARM64 @@ -537,7 +874,7 @@ Resources: awslogs-region: !Ref AWS::Region awslogs-stream-prefix: download - Name: migrate - Image: !Ref BackendImageUri + Image: !If [SeparateMigrationImage, !Ref MigrationImageUri, !Ref BackendImageUri] Essential: false User: '10001:10001' EntryPoint: [python, -m] @@ -630,7 +967,7 @@ Resources: - Key: Environment Value: demo - Key: TransportNotice - Value: temporary-http-not-production-ready + Value: accepted-temporary-http-cloudfront-to-alb FrontendTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup @@ -656,12 +993,138 @@ Resources: Port: 80 Protocol: HTTP DefaultActions: + - !If + - OriginLockdown + - Type: fixed-response + FixedResponseConfig: + ContentType: text/plain + MessageBody: Forbidden + StatusCode: '403' + - Type: forward + TargetGroupArn: !Ref FrontendTargetGroup + + OriginVerificationListenerRule: + Type: AWS::ElasticLoadBalancingV2::ListenerRule + Properties: + ListenerArn: !Ref HttpListener + Priority: 1 + Conditions: + - Field: http-header + HttpHeaderConfig: + HttpHeaderName: X-FIP-Origin-Verify + Values: [!Ref OriginVerificationToken] + Actions: - Type: forward TargetGroupArn: !Ref FrontendTargetGroup + DefaultOriginRequestPolicy: + Type: AWS::CloudFront::OriginRequestPolicy + Properties: + OriginRequestPolicyConfig: + Name: !Sub '${AWS::StackName}-default-origin' + Comment: Forward OAuth callback query strings without viewer Host or cookies + CookiesConfig: + CookieBehavior: none + HeadersConfig: + HeaderBehavior: none + QueryStringsConfig: + QueryStringBehavior: all + + CloudFrontDistribution: + Type: AWS::CloudFront::Distribution + DependsOn: OriginVerificationListenerRule + Properties: + DistributionConfig: + Comment: !Sub '${AWS::StackName} authenticated customer pilot' + Enabled: true + HttpVersion: http2and3 + IPV6Enabled: true + PriceClass: PriceClass_100 + DefaultRootObject: index.html + Origins: + - Id: AlbOrigin + DomainName: !GetAtt LoadBalancer.DNSName + ConnectionAttempts: 3 + ConnectionTimeout: 10 + OriginCustomHeaders: + - HeaderName: X-FIP-Origin-Verify + HeaderValue: !Ref OriginVerificationToken + CustomOriginConfig: + HTTPPort: 80 + OriginProtocolPolicy: http-only + OriginReadTimeout: 30 + OriginKeepaliveTimeout: 5 + DefaultCacheBehavior: + TargetOriginId: AlbOrigin + ViewerProtocolPolicy: redirect-to-https + AllowedMethods: [GET, HEAD, OPTIONS] + CachedMethods: [GET, HEAD, OPTIONS] + # AWS managed policy Managed-CachingDisabled. + CachePolicyId: 4135ea2d-6df8-44a3-9df3-4b5a84be39ad + OriginRequestPolicyId: !Ref DefaultOriginRequestPolicy + Compress: true + CacheBehaviors: + - PathPattern: /api/* + TargetOriginId: AlbOrigin + ViewerProtocolPolicy: redirect-to-https + AllowedMethods: [GET, HEAD, OPTIONS, PUT, PATCH, POST, DELETE] + CachedMethods: [GET, HEAD, OPTIONS] + # AWS managed policies Managed-CachingDisabled and + # Managed-AllViewerExceptHostHeader. The latter forwards + # Authorization while replacing the viewer Host for the ALB origin. + CachePolicyId: 4135ea2d-6df8-44a3-9df3-4b5a84be39ad + OriginRequestPolicyId: b689b0a8-53d0-40ab-baf2-68738e2966ac + Compress: true + ViewerCertificate: + CloudFrontDefaultCertificate: true + Tags: + - Key: Environment + Value: demo + + CognitoUserPoolClient: + Type: AWS::Cognito::UserPoolClient + Properties: + UserPoolId: !Ref CognitoUserPool + ClientName: !Sub '${AWS::StackName}-browser' + GenerateSecret: false + AllowedOAuthFlowsUserPoolClient: true + AllowedOAuthFlows: [code] + AllowedOAuthScopes: [openid, email, profile] + SupportedIdentityProviders: [COGNITO] + CallbackURLs: + - !Sub 'https://${CloudFrontDistribution.DomainName}/auth/callback' + LogoutURLs: + - !Sub 'https://${CloudFrontDistribution.DomainName}/' + DefaultRedirectURI: !Sub 'https://${CloudFrontDistribution.DomainName}/auth/callback' + EnableTokenRevocation: true + PreventUserExistenceErrors: ENABLED + AccessTokenValidity: 60 + IdTokenValidity: 60 + RefreshTokenValidity: 1 + TokenValidityUnits: + AccessToken: minutes + IdToken: minutes + RefreshToken: days + ReadAttributes: [email, email_verified] + + CognitoUserPoolDomain: + Type: AWS::Cognito::UserPoolDomain + Properties: + UserPoolId: !Ref CognitoUserPool + Domain: !Ref CognitoDomainPrefix + ManagedLoginVersion: 2 + + CognitoManagedLoginBranding: + Type: AWS::Cognito::ManagedLoginBranding + DependsOn: CognitoUserPoolDomain + Properties: + UserPoolId: !Ref CognitoUserPool + ClientId: !Ref CognitoUserPoolClient + UseCognitoProvidedValues: true + ApplicationService: Type: AWS::ECS::Service - DependsOn: HttpListener + DependsOn: OriginVerificationListenerRule Properties: ServiceName: application Cluster: !Ref EcsCluster @@ -690,8 +1153,22 @@ Resources: Outputs: TemporaryDemoUrl: - Description: HTTP-only temporary demo URL; do not treat as production-ready. - Value: !Sub 'http://${LoadBalancer.DNSName}' + Description: Legacy output name retained for automation; now points to CloudFront HTTPS. + Value: !Sub 'https://${CloudFrontDistribution.DomainName}' + LoadBalancerDnsName: + Description: Raw HTTP origin name for diagnostics; not the customer URL. + Value: !GetAtt LoadBalancer.DNSName + CloudFrontUrl: + Description: Public customer URL using the CloudFront default certificate. + Value: !Sub 'https://${CloudFrontDistribution.DomainName}' + CloudFrontDistributionId: + Value: !Ref CloudFrontDistribution + CognitoUserPoolId: + Value: !Ref CognitoUserPool + CognitoUserPoolClientId: + Value: !Ref CognitoUserPoolClient + CognitoManagedLoginDomain: + Value: !Sub 'https://${CognitoDomainPrefix}.auth.${AWS::Region}.amazoncognito.com' ClusterArn: Value: !GetAtt EcsCluster.Arn ApplicationServiceArn: @@ -706,5 +1183,19 @@ Outputs: Value: !Ref DatabaseInstanceClass ApplicationDesiredCount: Value: !Ref ApplicationDesiredCount + ApplicationDatabaseReaderUser: + Value: !Ref ApplicationDatabaseUser + ApplicationDatabaseWriterUser: + Value: !Ref ApplicationDatabaseWriterUser + ApplicationDatabaseWriterSecretArn: + Condition: RuntimeWriterConfigured + Value: !Ref ApplicationDatabaseWriterSecretArn + ApplicationDatabasePipelineUser: + Value: !Ref ApplicationDatabasePipelineUser + ApplicationDatabasePipelineSecretArn: + Condition: RuntimePipelineConfigured + Value: !Ref ApplicationDatabasePipelineSecretArn + WorkerEnabled: + Value: !Ref WorkerEnabled DatabaseSubnetIds: Value: !Join [',', [!Ref DatabaseSubnetA, !Ref DatabaseSubnetB]] diff --git a/infra/cloudformation/parameters.demo.example.json b/infra/cloudformation/parameters.demo.example.json index baa19fd..68c712a 100644 --- a/infra/cloudformation/parameters.demo.example.json +++ b/infra/cloudformation/parameters.demo.example.json @@ -3,6 +3,10 @@ "ParameterKey": "BackendImageUri", "ParameterValue": "208337080387.dkr.ecr.eu-west-1.amazonaws.com/foundation-intelligence-platform/backend:demo-ARM64-GIT_SHA" }, + { + "ParameterKey": "MigrationImageUri", + "ParameterValue": "" + }, { "ParameterKey": "FrontendImageUri", "ParameterValue": "208337080387.dkr.ecr.eu-west-1.amazonaws.com/foundation-intelligence-platform/frontend:demo-ARM64-GIT_SHA" @@ -15,6 +19,50 @@ "ParameterKey": "ApplicationDesiredCount", "ParameterValue": "0" }, + { + "ParameterKey": "ApplicationDatabaseWriterUser", + "ParameterValue": "foundation_app_writer" + }, + { + "ParameterKey": "ApplicationDatabaseWriterSecretArn", + "ParameterValue": "" + }, + { + "ParameterKey": "ApplicationDatabasePipelineUser", + "ParameterValue": "foundation_pipeline_writer" + }, + { + "ParameterKey": "ApplicationDatabasePipelineSecretArn", + "ParameterValue": "" + }, + { + "ParameterKey": "AnthropicSecretArn", + "ParameterValue": "arn:aws:secretsmanager:eu-west-1:208337080387:secret:foundation-intelligence/demo/anthropic-api-key-ABC123" + }, + { + "ParameterKey": "CharityCommissionSecretArn", + "ParameterValue": "arn:aws:secretsmanager:eu-west-1:208337080387:secret:foundation-intelligence/demo/charity-commission-api-key-ABC123" + }, + { + "ParameterKey": "WorkerEnabled", + "ParameterValue": "false" + }, + { + "ParameterKey": "OriginLockdownEnabled", + "ParameterValue": "false" + }, + { + "ParameterKey": "CloudFrontOriginPrefixListId", + "ParameterValue": "" + }, + { + "ParameterKey": "OriginVerificationToken", + "ParameterValue": "REPLACE_WITH_43_PLUS_BASE64URL_RANDOM_VALUE" + }, + { + "ParameterKey": "CognitoDomainPrefix", + "ParameterValue": "replace-with-available-fip-demo-prefix" + }, { "ParameterKey": "DatabaseInstanceClass", "ParameterValue": "db.t4g.medium" diff --git a/mypy.ini b/mypy.ini index adbffc4..9d271a0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,15 +12,30 @@ files = scripts/verify_local_rollback.py, src/bff/config.py, src/bff/database.py, + src/bff/main.py, + src/bff/auth.py, src/bff/security.py, + src/bff/user_management.py, + src/bff/postgres/admin_routes.py, + src/bff/postgres/dependencies.py, src/bff/postgres/governance_repository.py, + src/bff/postgres/governance_routes.py, + src/bff/postgres/idempotency_repository.py, src/bff/postgres/job_repository.py, src/bff/postgres/pipeline_repository.py, + src/bff/postgres/registry_repository.py, + src/bff/postgres/routes.py, + src/bff/postgres/scraper_routes.py, src/governance/exposure.py, src/governance/retention.py, + src/migration/aws_entrypoint.py, + src/migration/database_access.py, src/migration/sqlite_to_postgres.py, src/observability/metrics.py, src/transition/runtime.py, src/transition/shadow.py, src/pipelines/durable.py, - src/pipelines/durable_worker.py + src/pipelines/durable_worker.py, + src/pipelines/job_admin.py, + src/pipelines/worker.py, + src/pipelines/worker_handlers.py diff --git a/requirements-runtime.in b/requirements-runtime.in index b5a7cb6..84bfb41 100644 --- a/requirements-runtime.in +++ b/requirements-runtime.in @@ -4,6 +4,7 @@ alembic==1.18.5 anthropic==0.118.0 asyncpg==0.31.0 beautifulsoup4==4.15.0 +boto3==1.40.0 fastapi==0.139.2 feedparser==6.0.12 google-genai==2.14.0 diff --git a/requirements-runtime.txt b/requirements-runtime.txt index c390447..891285a 100644 --- a/requirements-runtime.txt +++ b/requirements-runtime.txt @@ -91,6 +91,16 @@ beautifulsoup4==4.15.0 \ --hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \ --hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9 # via -r requirements-runtime.in +boto3==1.40.0 \ + --hash=sha256:959443055d2af676c336cc6033b3f870a8a924384b70d0b2905081d649378179 \ + --hash=sha256:fc1b3ca3baf3d8820c6faddf47cbba8ad3cd16f8e8d7e2f76d304bf995932eb7 + # via -r requirements-runtime.in +botocore==1.40.76 \ + --hash=sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc \ + --hash=sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4 + # via + # boto3 + # s3transfer certifi==2026.7.22 \ --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 @@ -595,6 +605,12 @@ jiter==0.16.0 \ --hash=sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e \ --hash=sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de # via anthropic +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # boto3 + # botocore mako==1.3.12 \ --hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \ --hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a @@ -905,6 +921,10 @@ pysocks==1.7.1 \ --hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \ --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0 # via googlenewsdecoder +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore python-jose==3.5.0 \ --hash=sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771 \ --hash=sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b @@ -925,6 +945,10 @@ rsa==4.9.1 \ --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 # via python-jose +s3transfer==0.13.1 \ + --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 \ + --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf + # via boto3 selectolax==0.4.11 \ --hash=sha256:086ca6f7e4c475bfff871ec1448ae5d342d43d6a2ca2cea65160d01b3a6a75ec \ --hash=sha256:168b34466dc4f998d7ceb04f218693546d141543e7c5d327f9e006c0799cd62c \ @@ -997,7 +1021,9 @@ sgmllib3k==1.0.0 \ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 - # via ecdsa + # via + # ecdsa + # python-dateutil sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc @@ -1104,7 +1130,9 @@ typing-inspection==0.4.2 \ urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 - # via requests + # via + # botocore + # requests uvicorn==0.51.0 \ --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b \ --hash=sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0 diff --git a/requirements.txt b/requirements.txt index f21e4dc..3884ac0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -127,6 +127,16 @@ beautifulsoup4==4.15.0 \ --hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \ --hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9 # via -r requirements-runtime.in +boto3==1.40.0 \ + --hash=sha256:959443055d2af676c336cc6033b3f870a8a924384b70d0b2905081d649378179 \ + --hash=sha256:fc1b3ca3baf3d8820c6faddf47cbba8ad3cd16f8e8d7e2f76d304bf995932eb7 + # via -r requirements-runtime.in +botocore==1.40.76 \ + --hash=sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc \ + --hash=sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4 + # via + # boto3 + # s3transfer certifi==2026.7.22 \ --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 @@ -732,6 +742,12 @@ jiter==0.16.0 \ --hash=sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e \ --hash=sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de # via anthropic +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # boto3 + # botocore librt==0.13.0 \ --hash=sha256:05d96b80b95d3a2721b619f8982b8558848b04875bb4772fd54842b59f61dd97 \ --hash=sha256:0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b \ @@ -1227,6 +1243,10 @@ pytest-cov==7.1.0 \ --hash=sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2 \ --hash=sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678 # via -r requirements.in +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore python-jose==3.5.0 \ --hash=sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771 \ --hash=sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b @@ -1247,6 +1267,10 @@ rsa==4.9.1 \ --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 # via python-jose +s3transfer==0.13.1 \ + --hash=sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724 \ + --hash=sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf + # via boto3 selectolax==0.4.11 \ --hash=sha256:086ca6f7e4c475bfff871ec1448ae5d342d43d6a2ca2cea65160d01b3a6a75ec \ --hash=sha256:168b34466dc4f998d7ceb04f218693546d141543e7c5d327f9e006c0799cd62c \ @@ -1319,7 +1343,9 @@ sgmllib3k==1.0.0 \ six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 - # via ecdsa + # via + # ecdsa + # python-dateutil sniffio==1.3.1 \ --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc @@ -1427,7 +1453,9 @@ typing-inspection==0.4.2 \ urllib3==2.7.0 \ --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 - # via requests + # via + # botocore + # requests uvicorn==0.51.0 \ --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b \ --hash=sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0 diff --git a/src/bff/auth.py b/src/bff/auth.py index eb017e2..edce3c2 100644 --- a/src/bff/auth.py +++ b/src/bff/auth.py @@ -23,6 +23,40 @@ router = APIRouter(prefix="/api/auth", tags=["Authentication"]) +@router.get("/config") +async def auth_config(request: Request): + """Return only the public browser configuration required for managed login.""" + settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) + if settings.auth_mode != "cognito_rbac": + return {"mode": settings.auth_mode} + return { + "mode": "cognito_rbac", + "region": settings.cognito_region, + "user_pool_id": settings.cognito_user_pool_id, + "client_id": settings.cognito_client_id, + "domain": settings.cognito_domain, + "scopes": ["openid", "email", "profile"], + } + + +@router.get("/me") +async def current_user( + principal: Principal = Depends( + require_roles(Role.CUSTOMER, action="auth.me") + ), +): + payload = { + "sub": principal.actor_id, + "roles": sorted(role.value for role in principal.roles), + } + username = principal.claims.get("username") or principal.claims.get( + "cognito:username" + ) + if isinstance(username, str) and username.strip(): + payload["username"] = username.strip() + return payload + + @router.post("/login") async def login( request: Request, @@ -41,8 +75,8 @@ async def login( token = create_development_access_token(credentials.username, settings=settings) request.state.principal = Principal( actor_id=credentials.username, - roles=frozenset({Role.ADMINISTRATOR}), - claims={"sub": credentials.username, "roles": [Role.ADMINISTRATOR.value]}, + roles=frozenset({Role.ADMIN}), + claims={"sub": credentials.username, "roles": [Role.ADMIN.value]}, ) response.set_cookie( key=settings.session_cookie_name, @@ -61,7 +95,7 @@ async def logout( request: Request, response: Response, principal: Principal = Depends( - require_roles(Role.VIEWER, action="auth.logout") + require_roles(Role.CUSTOMER, action="auth.logout") ), ): """Clear the development session cookie; OIDC logout is owned by the IdP.""" diff --git a/src/bff/charity.py b/src/bff/charity.py index 66ae0ee..01f1aab 100644 --- a/src/bff/charity.py +++ b/src/bff/charity.py @@ -385,7 +385,9 @@ def _official_charity_url(charity_number: int) -> str: ) -def _fast_link_confirmed_profiles(reg_numbers: List[int]) -> List[Dict[str, Any]]: +def _fast_link_confirmed_profiles( + reg_numbers: List[int], database_path: Optional[str] = None +) -> List[Dict[str, Any]]: """Create lightweight profiles from cached official records and link known grants. This intentionally performs no network calls, database clone, currency @@ -395,10 +397,11 @@ def _fast_link_confirmed_profiles(reg_numbers: List[int]) -> List[Dict[str, Any] The operation is one small SQLite transaction so the directory can open a profile immediately after the user confirms it. """ - if not os.path.exists(DB_PATH): + target_database = database_path or DB_PATH + if not os.path.exists(target_database): raise RuntimeError("The local organization database is unavailable.") - conn = sqlite3.connect(DB_PATH, timeout=8) + conn = sqlite3.connect(target_database, timeout=8) conn.row_factory = sqlite3.Row try: conn.execute("PRAGMA foreign_keys = ON") @@ -1002,7 +1005,7 @@ async def get_default_charity_relevance_score( @router.post( "/{reg_charity_number}/score", response_model=ScoreResponse, - dependencies=[Depends(require_roles(Role.ANALYST, action="score.calculate"))], + dependencies=[Depends(require_roles(Role.OPERATOR, action="score.calculate"))], ) async def score_charity_relevance( reg_charity_number: int, diff --git a/src/bff/config.py b/src/bff/config.py index ace280d..ca4c163 100644 --- a/src/bff/config.py +++ b/src/bff/config.py @@ -6,6 +6,7 @@ import json import os from pathlib import Path +import re from typing import Mapping, Optional, Tuple from urllib.parse import urlparse @@ -76,6 +77,10 @@ class SecuritySettings: oidc_algorithms: Tuple[str, ...] oidc_role_claim: str oidc_jwks_cache_seconds: int + cognito_region: Optional[str] + cognito_user_pool_id: Optional[str] + cognito_client_id: Optional[str] + cognito_domain: Optional[str] dev_auth_enabled: bool dev_auth_username: Optional[str] dev_auth_password: Optional[str] @@ -115,6 +120,10 @@ def from_env(cls, environ: Optional[Mapping[str, str]] = None) -> "SecuritySetti oidc_algorithms=_as_csv(env.get("OIDC_ALGORITHMS"), "RS256"), oidc_role_claim=env.get("OIDC_ROLE_CLAIM", "roles").strip(), oidc_jwks_cache_seconds=_as_int(env.get("OIDC_JWKS_CACHE_SECONDS"), 300), + cognito_region=env.get("COGNITO_REGION"), + cognito_user_pool_id=env.get("COGNITO_USER_POOL_ID"), + cognito_client_id=env.get("COGNITO_CLIENT_ID"), + cognito_domain=env.get("COGNITO_DOMAIN"), dev_auth_enabled=_as_bool(env.get("DEV_AUTH_ENABLED")), dev_auth_username=env.get("DEV_AUTH_USERNAME"), dev_auth_password=env.get("DEV_AUTH_PASSWORD"), @@ -167,12 +176,18 @@ def validate_security_settings(settings: SecuritySettings) -> None: errors = [] production = settings.app_env in {"staging", "production"} - if settings.auth_mode not in {"disabled", "development", "oidc", "public_readonly"}: + if settings.auth_mode not in { + "disabled", + "development", + "oidc", + "public_readonly", + "cognito_rbac", + }: errors.append( - "AUTH_MODE must be disabled, development, oidc or public_readonly" + "AUTH_MODE must be disabled, development, oidc, public_readonly or cognito_rbac" ) - if production and settings.auth_mode != "oidc": - errors.append("staging/production requires AUTH_MODE=oidc") + if production and settings.auth_mode not in {"oidc", "cognito_rbac"}: + errors.append("staging/production requires AUTH_MODE=oidc or cognito_rbac") if production and settings.dev_auth_enabled: errors.append("development authentication must be disabled outside local/test environments") @@ -218,6 +233,40 @@ def validate_security_settings(settings: SecuritySettings) -> None: if value and urlparse(value).scheme != "https": errors.append(f"{name} must use HTTPS outside development") + if settings.auth_mode == "cognito_rbac": + if not all( + ( + settings.cognito_region, + settings.cognito_user_pool_id, + settings.cognito_client_id, + settings.cognito_domain, + ) + ): + errors.append( + "COGNITO_REGION, COGNITO_USER_POOL_ID, COGNITO_CLIENT_ID and " + "COGNITO_DOMAIN are required" + ) + if settings.cognito_region and not re.fullmatch( + r"[a-z]{2}(?:-gov)?-[a-z]+-\d", settings.cognito_region + ): + errors.append("COGNITO_REGION is invalid") + if settings.cognito_user_pool_id and not re.fullmatch( + r"[\w-]+_[0-9A-Za-z]+", settings.cognito_user_pool_id + ): + errors.append("COGNITO_USER_POOL_ID is invalid") + if settings.cognito_domain: + parsed_domain = urlparse(settings.cognito_domain) + if ( + parsed_domain.scheme != "https" + or not parsed_domain.hostname + or parsed_domain.path not in {"", "/"} + or parsed_domain.query + or parsed_domain.fragment + ): + errors.append("COGNITO_DOMAIN must be an HTTPS origin without path or query") + if settings.dev_auth_enabled: + errors.append("development authentication must be disabled in cognito_rbac mode") + if production and not settings.session_cookie_secure: errors.append("SESSION_COOKIE_SECURE must be true outside development") if production and not settings.cors_origins: diff --git a/src/bff/database.py b/src/bff/database.py index 03788e1..0b45c49 100644 --- a/src/bff/database.py +++ b/src/bff/database.py @@ -21,6 +21,10 @@ class DatabaseConfigurationError(RuntimeError): """Raised when PostgreSQL configuration is incomplete or unsafe.""" +class WriterDatabaseUnavailable(RuntimeError): + """Raised when an explicit mutation has no configured writer principal.""" + + def _positive_int(value: Optional[str], default: int) -> int: parsed = int(value) if value is not None else default if parsed <= 0: @@ -33,6 +37,7 @@ def _ssl_mode(environment: Mapping[str, str]) -> str: explicit = environment.get("DATABASE_SSL_MODE") url = environment.get("DATABASE_URL") query_modes: set[str] = set() + query: Mapping[str, str | tuple[str, ...]] = {} if url: try: query = make_url(url).query @@ -99,6 +104,100 @@ def from_env(cls, environ: Optional[Mapping[str, str]] = None) -> "DatabaseSetti statement_timeout_ms=_positive_int(env.get("DATABASE_STATEMENT_TIMEOUT_MS"), 30_000), ) + @classmethod + def writer_from_env( + cls, environ: Optional[Mapping[str, str]] = None + ) -> Optional["DatabaseSettings"]: + """Build an independent writer configuration without credential fallback. + + Network and pool settings may inherit the reader connection's non-secret + values. Writer identity and password must always be provided through the + dedicated ``DATABASE_WRITE_*`` namespace. + """ + env = os.environ if environ is None else environ + writer_credentials = { + "DATABASE_WRITE_URL", + "DATABASE_WRITE_USER", + "DATABASE_WRITE_PASSWORD", + "DATABASE_WRITE_PASSWORD_FILE", + } + if not any(env.get(name) for name in writer_credentials): + return None + normalized = { + "DATABASE_URL": env.get("DATABASE_WRITE_URL"), + "DATABASE_HOST": env.get("DATABASE_WRITE_HOST") or env.get("DATABASE_HOST"), + "DATABASE_PORT": env.get("DATABASE_WRITE_PORT") or env.get("DATABASE_PORT"), + "DATABASE_NAME": env.get("DATABASE_WRITE_NAME") or env.get("DATABASE_NAME"), + "DATABASE_USER": env.get("DATABASE_WRITE_USER"), + "DATABASE_PASSWORD": env.get("DATABASE_WRITE_PASSWORD"), + "DATABASE_PASSWORD_FILE": env.get("DATABASE_WRITE_PASSWORD_FILE"), + "DATABASE_SSL_MODE": env.get("DATABASE_WRITE_SSL_MODE") + or env.get("DATABASE_SSL_MODE"), + "DATABASE_POOL_SIZE": env.get("DATABASE_WRITE_POOL_SIZE") or "3", + "DATABASE_MAX_OVERFLOW": env.get("DATABASE_WRITE_MAX_OVERFLOW") or "2", + "DATABASE_POOL_TIMEOUT_SECONDS": env.get( + "DATABASE_WRITE_POOL_TIMEOUT_SECONDS" + ) + or env.get("DATABASE_POOL_TIMEOUT_SECONDS"), + "DATABASE_CONNECT_TIMEOUT_SECONDS": env.get( + "DATABASE_WRITE_CONNECT_TIMEOUT_SECONDS" + ) + or env.get("DATABASE_CONNECT_TIMEOUT_SECONDS"), + "DATABASE_STATEMENT_TIMEOUT_MS": env.get( + "DATABASE_WRITE_STATEMENT_TIMEOUT_MS" + ) + or env.get("DATABASE_STATEMENT_TIMEOUT_MS"), + } + settings = cls.from_env( + {key: value for key, value in normalized.items() if value is not None} + ) + if not settings.configured: + raise DatabaseConfigurationError( + "Writer PostgreSQL credentials are incomplete; no reader credential fallback is allowed" + ) + return settings + + @classmethod + def pipeline_from_env( + cls, environ: Optional[Mapping[str, str]] = None + ) -> Optional["DatabaseSettings"]: + """Build the dataset-publisher connection without runtime-writer fallback.""" + env = os.environ if environ is None else environ + pipeline_credentials = { + "DATABASE_PIPELINE_URL", + "DATABASE_PIPELINE_USER", + "DATABASE_PIPELINE_PASSWORD", + "DATABASE_PIPELINE_PASSWORD_FILE", + } + if not any(env.get(name) for name in pipeline_credentials): + return None + normalized = { + "DATABASE_URL": env.get("DATABASE_PIPELINE_URL"), + "DATABASE_HOST": env.get("DATABASE_PIPELINE_HOST") or env.get("DATABASE_HOST"), + "DATABASE_PORT": env.get("DATABASE_PIPELINE_PORT") or env.get("DATABASE_PORT"), + "DATABASE_NAME": env.get("DATABASE_PIPELINE_NAME") or env.get("DATABASE_NAME"), + "DATABASE_USER": env.get("DATABASE_PIPELINE_USER"), + "DATABASE_PASSWORD": env.get("DATABASE_PIPELINE_PASSWORD"), + "DATABASE_PASSWORD_FILE": env.get("DATABASE_PIPELINE_PASSWORD_FILE"), + "DATABASE_SSL_MODE": env.get("DATABASE_PIPELINE_SSL_MODE") + or env.get("DATABASE_SSL_MODE"), + "DATABASE_POOL_SIZE": "1", + "DATABASE_MAX_OVERFLOW": "1", + "DATABASE_POOL_TIMEOUT_SECONDS": env.get("DATABASE_POOL_TIMEOUT_SECONDS"), + "DATABASE_CONNECT_TIMEOUT_SECONDS": env.get( + "DATABASE_CONNECT_TIMEOUT_SECONDS" + ), + "DATABASE_STATEMENT_TIMEOUT_MS": "86400000", + } + settings = cls.from_env( + {key: value for key, value in normalized.items() if value is not None} + ) + if not settings.configured: + raise DatabaseConfigurationError( + "Pipeline PostgreSQL credentials are incomplete; no runtime credential fallback is allowed" + ) + return settings + @property def configured(self) -> bool: return bool( @@ -341,3 +440,83 @@ async def close(self) -> None: if self._health_engine is not None: await self._health_engine.dispose() self._health_engine = None + + +class DatabaseAccess: + """Separate reader, API-writer and worker-publisher connection pools.""" + + def __init__( + self, + reader_settings: DatabaseSettings, + writer_settings: Optional[DatabaseSettings] = None, + pipeline_settings: Optional[DatabaseSettings] = None, + ) -> None: + self.reader = DatabaseManager(reader_settings) + self.writer = DatabaseManager(writer_settings) if writer_settings else None + self.pipeline = DatabaseManager(pipeline_settings) if pipeline_settings else None + + @classmethod + def from_env( + cls, environ: Optional[Mapping[str, str]] = None + ) -> "DatabaseAccess": + return cls( + DatabaseSettings.from_env(environ), + DatabaseSettings.writer_from_env(environ), + DatabaseSettings.pipeline_from_env(environ), + ) + + @property + def configured(self) -> bool: + return self.reader.configured + + @property + def writer_configured(self) -> bool: + return bool(self.writer and self.writer.configured) + + @property + def pipeline_configured(self) -> bool: + return bool(self.pipeline and self.pipeline.configured) + + def sessions(self) -> async_sessionmaker[AsyncSession]: + """Return the reader factory used by startup, health and all GET paths.""" + return self.reader.sessions() + + def write_sessions(self) -> async_sessionmaker[AsyncSession]: + """Return the writer factory only for an explicitly authorized mutation.""" + if not self.writer_configured or self.writer is None: + raise WriterDatabaseUnavailable("The runtime writer is not configured") + return self.writer.sessions() + + def pipeline_sessions(self) -> async_sessionmaker[AsyncSession]: + """Return the worker-only dataset publisher session factory.""" + if not self.pipeline_configured or self.pipeline is None: + raise WriterDatabaseUnavailable("The pipeline publisher is not configured") + return self.pipeline.sessions() + + async def check(self) -> bool: + return await self.reader.check() + + async def readiness( + self, + *, + expected_schema_version: str, + require_critical_configuration: bool = True, + ) -> dict[str, object]: + return await self.reader.readiness( + expected_schema_version=expected_schema_version, + require_critical_configuration=require_critical_configuration, + ) + + async def writer_ready(self) -> bool: + """Return non-sensitive writer availability without affecting app readiness.""" + return bool(self.writer and await self.writer.check()) + + def pool_status(self) -> dict[str, float]: + return self.reader.pool_status() + + async def close(self) -> None: + await self.reader.close() + if self.writer is not None: + await self.writer.close() + if self.pipeline is not None: + await self.pipeline.close() diff --git a/src/bff/main.py b/src/bff/main.py index 3fcbf0a..f3868f1 100644 --- a/src/bff/main.py +++ b/src/bff/main.py @@ -10,9 +10,10 @@ from bff.auth import router as auth_router from bff.proxy import router as proxy_router from bff.news import router as news_router +from bff.user_management import router as user_management_router from bff.audit import StructuredLogAuditSink, event_from_request from bff.config import SECURITY_SETTINGS, validate_security_settings -from bff.database import DatabaseManager, DatabaseSettings +from bff.database import DatabaseAccess from bff.security import IdempotencyStore, SlidingWindowRateLimiter from bff.utils.logging import logger, pseudonymous_actor_id from observability.metrics import MetricsRegistry, load_observability_configuration @@ -29,16 +30,15 @@ POSTGRESQL_ONLY_RUNTIME = TRANSITION_SETTINGS.postgresql_authoritative if POSTGRESQL_ONLY_RUNTIME: from bff.postgres.admin_routes import router as admin_router - from bff.postgres.audit_repository import PostgresAuditSink from bff.postgres.base import ANALYTICS_CACHE - from bff.postgres.governance_repository import GovernanceRepository from bff.postgres.governance_routes import router as governance_router - from bff.postgres.idempotency_repository import PostgresIdempotencyStore + from bff.postgres.idempotency_repository import ( + PostgresIdempotencyStore, + UnavailablePostgresIdempotencyStore, + ) from bff.postgres.observability_routes import router as observability_router - from bff.postgres.pipeline_repository import PipelineRepository from bff.postgres.routes import router as charity_router - from governance.retention import load_governance_configuration - from pipelines.durable import load_source_configurations + from bff.postgres.scraper_routes import router as scraper_router else: # The legacy SQLite repository is restricted to development/test while the # remaining domain journeys are ported in Phase 5. @@ -76,39 +76,22 @@ def resolve_shadow_journey(method, path, query_string): async def lifespan(application: FastAPI): """Validate security before accepting traffic, then initialize repository state.""" validate_security_settings(application.state.security_settings) - application.state.database = DatabaseManager(DatabaseSettings.from_env()) + application.state.database = DatabaseAccess.from_env() if POSTGRESQL_ONLY_RUNTIME: - # Constructing the pool remains lazy; readiness owns the first bounded - # connection and all production repositories use the same manager. - sessions = application.state.database.sessions() - public_readonly = ( - application.state.security_settings.auth_mode == "public_readonly" - ) - if public_readonly: - # The public web task has a database role with SELECT only. It must - # never synchronize configuration or persist audit/idempotency rows - # as a side effect of startup or anonymous reads. - application.state.audit_sink = StructuredLogAuditSink() - application.state.idempotency_store = IdempotencyStore() - logger.info("Public read-only PostgreSQL runtime initialized without writes.") - else: - application.state.audit_sink = PostgresAuditSink(sessions) - application.state.idempotency_store = PostgresIdempotencyStore(sessions) - synchronized_sources = await PipelineRepository(sessions).synchronize_sources( - load_source_configurations() - ) - synchronized_policies = await GovernanceRepository(sessions).synchronize_policies( - load_governance_configuration() - ) - logger.info( - "Synchronized %s governance-gated source configurations.", - synchronized_sources, + # Pool construction is lazy and startup performs no SQL. All defaults, + # role grants and schema work belong to the explicit release task. + application.state.database.sessions() + application.state.audit_sink = StructuredLogAuditSink() + if application.state.database.writer_configured: + application.state.idempotency_store = PostgresIdempotencyStore( + application.state.database.write_sessions() ) - logger.info( - "Synchronized %s non-destructive retention policies.", - synchronized_policies, - ) - logger.info("PostgreSQL repository runtime initialized.") + else: + application.state.idempotency_store = UnavailablePostgresIdempotencyStore() + logger.info( + "PostgreSQL runtime initialized without startup DML; writer_configured=%s.", + application.state.database.writer_configured, + ) else: get_charity_repository() logger.info( @@ -131,6 +114,9 @@ async def lifespan(application: FastAPI): ), version="1.0.0", lifespan=lifespan, + docs_url=None if SECURITY_SETTINGS.auth_mode == "cognito_rbac" else "/docs", + redoc_url=None if SECURITY_SETTINGS.auth_mode == "cognito_rbac" else "/redoc", + openapi_url=None if SECURITY_SETTINGS.auth_mode == "cognito_rbac" else "/openapi.json", ) app.state.security_settings = SECURITY_SETTINGS @@ -140,7 +126,7 @@ async def lifespan(application: FastAPI): ) app.state.idempotency_store = IdempotencyStore() app.state.audit_sink = StructuredLogAuditSink() -app.state.database = DatabaseManager(DatabaseSettings.from_env()) +app.state.database = DatabaseAccess.from_env() app.state.observability_configuration = load_observability_configuration() app.state.metrics_registry = MetricsRegistry(app.state.observability_configuration) @@ -154,8 +140,8 @@ async def lifespan(application: FastAPI): app.add_middleware( CORSMiddleware, allow_origins=list(SECURITY_SETTINGS.cors_origins), - allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "OPTIONS"], + allow_credentials=SECURITY_SETTINGS.auth_mode == "development", + allow_methods=["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], allow_headers=[ "Accept", "Authorization", @@ -347,13 +333,17 @@ async def global_exception_handler(request: Request, exc: Exception): app.include_router(proxy_router) app.include_router(admin_router) if POSTGRESQL_ONLY_RUNTIME: + app.include_router(scraper_router) app.include_router(governance_router) app.include_router(observability_router) app.include_router(news_router) +app.include_router(user_management_router) @app.get("/", include_in_schema=False) async def root_redirect(): """Redirect root access to Swagger interactive documentation.""" + if app.state.security_settings.auth_mode == "cognito_rbac": + return JSONResponse(status_code=404, content={"detail": "Not found"}) return RedirectResponse(url="/docs") diff --git a/src/bff/news.py b/src/bff/news.py index 026e461..759a3ac 100644 --- a/src/bff/news.py +++ b/src/bff/news.py @@ -895,7 +895,7 @@ def _parse_news_aliases(aliases: Optional[str]) -> list[str]: router = APIRouter( prefix="/api/news", tags=["Foundation News"], - dependencies=[Depends(require_roles(Role.ANALYST, action="news.read"))], + dependencies=[Depends(require_roles(Role.OPERATOR, action="news.read"))], ) diff --git a/src/bff/postgres/admin_routes.py b/src/bff/postgres/admin_routes.py index 648cb88..9d2d910 100644 --- a/src/bff/postgres/admin_routes.py +++ b/src/bff/postgres/admin_routes.py @@ -3,11 +3,13 @@ from __future__ import annotations import json +import uuid from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from bff.postgres.job_repository import PIPELINE_JOB_TYPES, PostgresJobRepository from bff.postgres.pipeline_repository import PipelineRepository +from bff.postgres.dependencies import reader_sessions, writer_sessions from bff.schemas import PipelineStatus, PipelineTrigger, SourceScheduleUpdate from bff.security import Role, require_roles from bff.utils.logging import redact_text @@ -25,7 +27,11 @@ def _jobs(request: Request) -> PostgresJobRepository: - return PostgresJobRepository(request.app.state.database.sessions()) + return PostgresJobRepository(reader_sessions(request)) + + +def _write_jobs(request: Request) -> PostgresJobRepository: + return PostgresJobRepository(writer_sessions(request)) def _actor(request: Request) -> str: @@ -34,7 +40,11 @@ def _actor(request: Request) -> str: def _pipelines(request: Request) -> PipelineRepository: - return PipelineRepository(request.app.state.database.sessions()) + return PipelineRepository(reader_sessions(request)) + + +def _write_pipelines(request: Request) -> PipelineRepository: + return PipelineRepository(writer_sessions(request)) @router.get("/pipeline/status", response_model=PipelineStatus) @@ -52,7 +62,7 @@ async def get_pipeline_status( async def trigger_pipeline( payload: PipelineTrigger, request: Request, - repository: PostgresJobRepository = Depends(_jobs), + repository: PostgresJobRepository = Depends(_write_jobs), ): if payload.source not in PIPELINE_JOB_TYPES[:4]: raise HTTPException( @@ -64,6 +74,9 @@ async def trigger_pipeline( payload.model_dump(exclude={"source"}), actor_id=_actor(request), idempotency_key=str(request.headers.get("idempotency-key") or "").strip(), + active_dedupe_key=f"pipeline:{payload.source}", + max_attempts=1, + timeout_seconds=21_600 if payload.source == "full_run" else 3_600, ) return { "status": job["status"], @@ -96,6 +109,17 @@ async def get_pipeline_jobs( } +@router.get("/pipeline/jobs/{job_id}") +async def get_pipeline_job( + job_id: uuid.UUID, + repository: PostgresJobRepository = Depends(_jobs), +): + job = await repository.get(str(job_id)) + if job is None: + raise HTTPException(status_code=404, detail="Pipeline job not found") + return job + + @router.get( "/pipeline/logs", dependencies=[Depends(require_roles(Role.ADMINISTRATOR, action="pipeline.logs.read"))], @@ -148,7 +172,7 @@ async def get_pipeline_sources( async def update_pipeline_source_schedule( source_name: str, payload: SourceScheduleUpdate, - repository: PipelineRepository = Depends(_pipelines), + repository: PipelineRepository = Depends(_write_pipelines), ): try: return await repository.set_source_enabled(source_name, enabled=payload.enabled) diff --git a/src/bff/postgres/dependencies.py b/src/bff/postgres/dependencies.py new file mode 100644 index 0000000..1907783 --- /dev/null +++ b/src/bff/postgres/dependencies.py @@ -0,0 +1,26 @@ +"""Fail-closed read/write PostgreSQL dependency selection.""" + +from __future__ import annotations + +from fastapi import HTTPException, Request, status + +from bff.database import WriterDatabaseUnavailable + + +def reader_sessions(request: Request): + """Return the SELECT-only pool used by reads and pure calculations.""" + return request.app.state.database.sessions() + + +def writer_sessions(request: Request): + """Return the writer pool only after route-level RBAC dependencies pass.""" + try: + return request.app.state.database.write_sessions() + except WriterDatabaseUnavailable as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="The mutation database path is temporarily unavailable.", + ) from exc + + +__all__ = ["reader_sessions", "writer_sessions"] diff --git a/src/bff/postgres/funder_repository.py b/src/bff/postgres/funder_repository.py index 72d42cd..35c6394 100644 --- a/src/bff/postgres/funder_repository.py +++ b/src/bff/postgres/funder_repository.py @@ -10,6 +10,7 @@ from sqlalchemy import text from bff.postgres.base import PostgresRepository, iso_value, number_value, utc_now +from bff.postgres.organization_repository import OrganizationRepository class SourceFunderRepository(PostgresRepository): @@ -1135,3 +1136,65 @@ async def profile_cache(self, source_funder_key: str) -> Optional[dict[str, Any] "job_id": str(row["job_token"]) if row["job_token"] else None, "link_revision": int(row["link_revision"]), } + + async def hydrate_profile_cache( + self, source_funder_key: str, *, job_id: str + ) -> dict[str, Any]: + """Fill the current pending cache from the existing profile detail view.""" + key = str(source_funder_key or "").strip() + if not key or len(key) > 500: + raise ValueError("source_funder_key must be a non-empty canonical key.") + token = uuid.UUID(str(job_id)) + async with self.sessions() as session: + pending = ( + await session.execute( + text( + """ + SELECT dataset_version, profile_id, link_revision + FROM source_funder_profile_cache + WHERE source_funder_key=:key AND job_token=:job_id + AND status='pending' + """ + ), + {"key": key, "job_id": token}, + ) + ).mappings().first() + if pending is None: + raise ValueError("The pending profile hydration is no longer current") + + payload = await OrganizationRepository(self.sessions).detail( + int(pending["profile_id"]) + ) + if payload is None: + raise LookupError("No active organization profile is available for hydration") + + async with self.sessions() as session, session.begin(): + updated = await session.scalar( + text( + """ + UPDATE source_funder_profile_cache + SET status='ready', payload=CAST(:payload AS jsonb), error=NULL, + updated_at=CURRENT_TIMESTAMP + WHERE dataset_version=:dataset_version + AND source_funder_key=:key AND profile_id=:profile_id + AND link_revision=:link_revision AND job_token=:job_id + AND status='pending' + RETURNING source_funder_key + """ + ), + { + "dataset_version": pending["dataset_version"], + "key": key, + "profile_id": pending["profile_id"], + "link_revision": pending["link_revision"], + "job_id": token, + "payload": json.dumps(payload, sort_keys=True, default=str), + }, + ) + if updated is None: + raise ValueError("The pending profile hydration changed before completion") + return { + "source_funder_key": key, + "profile_id": int(pending["profile_id"]), + "status": "ready", + } diff --git a/src/bff/postgres/governance_routes.py b/src/bff/postgres/governance_routes.py index b56fa19..7049da7 100644 --- a/src/bff/postgres/governance_routes.py +++ b/src/bff/postgres/governance_routes.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request from bff.postgres.governance_repository import GovernanceRepository +from bff.postgres.dependencies import reader_sessions, writer_sessions from bff.schemas import ( DataHoldRelease, DataHoldRequest, @@ -33,7 +34,11 @@ def _repository(request: Request) -> GovernanceRepository: - return GovernanceRepository(request.app.state.database.sessions()) + return GovernanceRepository(reader_sessions(request)) + + +def _write_repository(request: Request) -> GovernanceRepository: + return GovernanceRepository(writer_sessions(request)) def _actor(request: Request) -> str: @@ -67,7 +72,7 @@ async def retention_policies( async def retention_dry_run( payload: RetentionDryRunRequest, request: Request, - repository: GovernanceRepository = Depends(_repository), + repository: GovernanceRepository = Depends(_write_repository), ): candidate = RetentionCandidate( target_type=payload.target_type, @@ -124,7 +129,7 @@ async def active_holds( async def create_hold( payload: DataHoldRequest, request: Request, - repository: GovernanceRepository = Depends(_repository), + repository: GovernanceRepository = Depends(_write_repository), ): try: return await repository.create_hold( @@ -155,7 +160,7 @@ async def release_hold( hold_id: str, payload: DataHoldRelease, request: Request, - repository: GovernanceRepository = Depends(_repository), + repository: GovernanceRepository = Depends(_write_repository), ): try: return await repository.release_hold( @@ -192,7 +197,7 @@ async def export_expiration_report( ) async def create_data_subject_request( payload: DataSubjectRequestCreate, - repository: GovernanceRepository = Depends(_repository), + repository: GovernanceRepository = Depends(_write_repository), ): request_id = await repository.create_data_subject_request( request_type=payload.request_type, diff --git a/src/bff/postgres/idempotency_repository.py b/src/bff/postgres/idempotency_repository.py index 0a2f9d8..52ef1fb 100644 --- a/src/bff/postgres/idempotency_repository.py +++ b/src/bff/postgres/idempotency_repository.py @@ -8,6 +8,7 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from bff.database import WriterDatabaseUnavailable from bff.security import IdempotencyConflict @@ -118,3 +119,19 @@ async def release(self, record_key: RecordKey) -> None: ), {"actor_id": actor_id, "action": action, "idempotency_key": key}, ) + + +class UnavailablePostgresIdempotencyStore: + """Fail closed when mutations are attempted without the writer pool.""" + + @staticmethod + async def reserve(*_args, **_kwargs): + raise WriterDatabaseUnavailable("The runtime writer is unavailable") + + @staticmethod + async def complete(_record_key: RecordKey) -> None: + raise WriterDatabaseUnavailable("The runtime writer is unavailable") + + @staticmethod + async def release(_record_key: RecordKey) -> None: + raise WriterDatabaseUnavailable("The runtime writer is unavailable") diff --git a/src/bff/postgres/job_repository.py b/src/bff/postgres/job_repository.py index ad40a2e..b5b30d2 100644 --- a/src/bff/postgres/job_repository.py +++ b/src/bff/postgres/job_repository.py @@ -33,6 +33,7 @@ async def enqueue( queue_name: str = "pipeline", max_attempts: int = 3, timeout_seconds: int = 3600, + active_dedupe_key: str | None = None, ) -> dict[str, Any]: job_type = str(job_type).strip() if not job_type or len(job_type) > 120: @@ -46,6 +47,13 @@ async def enqueue( raise ValueError("max_attempts must be between 1 and 20") if not 1 <= int(timeout_seconds) <= 86_400: raise ValueError("timeout_seconds must be between 1 and 86400") + normalized_dedupe_key = ( + str(active_dedupe_key).strip() if active_dedupe_key is not None else None + ) + if normalized_dedupe_key is not None and not normalized_dedupe_key: + normalized_dedupe_key = None + if normalized_dedupe_key and len(normalized_dedupe_key) > 200: + raise ValueError("active_dedupe_key must not exceed 200 characters") async with self.sessions() as session, session.begin(): dataset_version = await self.active_dataset(session) existing = ( @@ -68,33 +76,74 @@ async def enqueue( "idempotent_noop": True, } job_id = uuid.uuid4() - requested_at = await session.scalar( - text( + inserted = ( + await session.execute( + text( """ INSERT INTO job_runs ( job_run_id, job_type, status, dataset_version, idempotency_key, requested_by, input, queue_name, - max_attempts, timeout_seconds, last_good_dataset_version + max_attempts, timeout_seconds, last_good_dataset_version, + active_dedupe_key ) VALUES ( :job_id, :job_type, 'queued', :dataset_version, :idempotency_key, :actor_id, CAST(:payload AS jsonb), - :queue_name, :max_attempts, :timeout_seconds, :dataset_version + :queue_name, :max_attempts, :timeout_seconds, :dataset_version, + :active_dedupe_key ) + ON CONFLICT DO NOTHING RETURNING requested_at """ - ), - { - "job_id": job_id, - "job_type": job_type, - "dataset_version": dataset_version, - "idempotency_key": idempotency_key, - "actor_id": actor_id, - "payload": json.dumps(payload, sort_keys=True, default=str), - "queue_name": queue_name, - "max_attempts": int(max_attempts), - "timeout_seconds": int(timeout_seconds), - }, - ) + ), + { + "job_id": job_id, + "job_type": job_type, + "dataset_version": dataset_version, + "idempotency_key": idempotency_key, + "actor_id": actor_id, + "payload": json.dumps(payload, sort_keys=True, default=str), + "queue_name": queue_name, + "max_attempts": int(max_attempts), + "timeout_seconds": int(timeout_seconds), + "active_dedupe_key": normalized_dedupe_key, + }, + ) + ).mappings().first() + if inserted is None: + active = ( + await session.execute( + text( + """ + SELECT job_run_id, status, requested_at + FROM job_runs + WHERE (job_type=:job_type AND idempotency_key=:idempotency_key) + OR ( + :active_dedupe_key IS NOT NULL + AND active_dedupe_key=:active_dedupe_key + AND status IN ('queued', 'running') + ) + ORDER BY + CASE WHEN status IN ('queued', 'running') THEN 0 ELSE 1 END, + requested_at DESC + LIMIT 1 + """ + ), + { + "job_type": job_type, + "idempotency_key": idempotency_key, + "active_dedupe_key": normalized_dedupe_key, + }, + ) + ).mappings().first() + if active is None: + raise RuntimeError("Job enqueue conflict could not be reconciled") + return { + "job_id": str(active["job_run_id"]), + "status": str(active["status"]), + "requested_at": iso_value(active["requested_at"]), + "idempotent_noop": True, + } + requested_at = inserted["requested_at"] envelope = QueueEnvelope( schema_version="job-envelope-v1", job_id=job_id, @@ -186,7 +235,8 @@ async def claim( RETURNING run.job_run_id, run.job_type, run.input, run.dataset_version, run.requested_at, run.started_at, run.attempt, run.max_attempts, - run.timeout_seconds, run.last_good_dataset_version + run.timeout_seconds, run.last_good_dataset_version, + run.requested_by """ ), {"queue_name": queue_name, "lease_seconds": int(lease_seconds)}, @@ -216,6 +266,21 @@ async def claim( ) if not row: return None + await session.execute( + text( + """ + UPDATE job_dispatch_outbox + SET status='published', publish_attempts=publish_attempts + 1, + queue_message_id=:queue_message_id, + published_at=CURRENT_TIMESTAMP, last_error_class=NULL + WHERE job_run_id=:job_id AND status IN ('pending', 'failed') + """ + ), + { + "job_id": row["job_run_id"], + "queue_message_id": f"postgres-worker:{worker_id}"[:200], + }, + ) await self._event( session, row["job_run_id"], @@ -275,7 +340,11 @@ async def succeed( SET status='succeeded', completed_at=CURRENT_TIMESTAMP, heartbeat_at=CURRENT_TIMESTAMP, lease_expires_at=NULL, result=CAST(:result AS jsonb), error_class=NULL, - error_message=NULL, failure_reason=NULL + error_message=NULL, failure_reason=NULL, + dataset_version=COALESCE(:result_dataset_version, dataset_version), + last_good_dataset_version=COALESCE( + :result_dataset_version, last_good_dataset_version + ) WHERE job_run_id=:job_id AND status='running' RETURNING job_run_id """ @@ -283,6 +352,7 @@ async def succeed( { "job_id": uuid.UUID(str(job_id)), "result": json.dumps(result, sort_keys=True, default=str), + "result_dataset_version": result.get("dataset_version"), }, ) if updated: @@ -340,6 +410,19 @@ async def fail( "failure_reason": str(failure_reason)[:2000], }, ) + if not will_retry: + await session.execute( + text( + """ + UPDATE source_funder_profile_cache + SET status='failed', payload=NULL, + error='Profile hydration could not be completed.', + updated_at=CURRENT_TIMESTAMP + WHERE job_token=:job_id AND status='pending' + """ + ), + {"job_id": row["job_run_id"]}, + ) await self._event( session, row["job_run_id"], @@ -350,55 +433,79 @@ async def fail( await self._idle_worker(session, worker_id) return status_value - async def requeue_expired(self, *, queue_name: str = "pipeline") -> dict[str, int]: + async def fail_expired(self, *, queue_name: str = "pipeline") -> dict[str, int]: async with self.sessions() as session, session.begin(): - retry_rows = ( + failed_rows = ( await session.execute( text( """ UPDATE job_runs - SET status='queued', attempt=attempt + 1, + SET status='failed', completed_at=CURRENT_TIMESTAMP, heartbeat_at=NULL, lease_expires_at=NULL, error_class='WorkerLeaseExpired', - error_message='Worker heartbeat lease expired', + error_message='Worker heartbeat lease expired; automatic retry is disabled', failure_reason='worker_lease_expired' WHERE queue_name=:queue_name AND status='running' AND lease_expires_at < CURRENT_TIMESTAMP - AND attempt < max_attempts RETURNING job_run_id """ ), {"queue_name": queue_name}, ) ).scalars().all() - dead_rows = ( + for job_id in failed_rows: + await self._event( + session, job_id, "failed", "lease-reaper", {"reason": "lease_expired"} + ) + if failed_rows: await session.execute( text( """ - UPDATE job_runs - SET status='dead_lettered', completed_at=CURRENT_TIMESTAMP, - heartbeat_at=NULL, lease_expires_at=NULL, - error_class='WorkerLeaseExpired', - error_message='Worker heartbeat lease expired after final attempt', - failure_reason='worker_lease_expired' - WHERE queue_name=:queue_name AND status='running' - AND lease_expires_at < CURRENT_TIMESTAMP - AND attempt >= max_attempts - RETURNING job_run_id + UPDATE source_funder_profile_cache + SET status='failed', payload=NULL, + error='Profile hydration could not be completed.', + updated_at=CURRENT_TIMESTAMP + WHERE job_token=ANY(CAST(:job_ids AS uuid[])) + AND status='pending' """ ), - {"queue_name": queue_name}, + {"job_ids": list(failed_rows)}, ) - ).scalars().all() - for job_id in retry_rows: - await self._event( - session, job_id, "retry_queued", "lease-reaper", {"reason": "lease_expired"} + await session.execute( + text( + """ + UPDATE worker_heartbeats + SET status='stopped', job_run_id=NULL, + heartbeat_at=CURRENT_TIMESTAMP + WHERE job_run_id=ANY(CAST(:job_ids AS uuid[])) + """ + ), + {"job_ids": list(failed_rows)}, ) - for job_id in dead_rows: - await self._event( - session, job_id, "dead_lettered", "lease-reaper", {"reason": "lease_expired"} + return {"failed": len(failed_rows)} + + async def requeue_expired(self, *, queue_name: str = "pipeline") -> dict[str, int]: + """Backward-compatible name; expired work now fails without retry.""" + result = await self.fail_expired(queue_name=queue_name) + return {"requeued": 0, "dead_lettered": 0, **result} + + async def get(self, job_id: str) -> dict[str, Any] | None: + async with self.sessions() as session: + row = ( + await session.execute( + text( + """ + SELECT job_run_id, job_type, status, dataset_version, + requested_at, started_at, completed_at, heartbeat_at, + attempt, max_attempts, result, error_class, + error_message, failure_reason + FROM job_runs WHERE job_run_id=:job_id + """ + ), + {"job_id": uuid.UUID(str(job_id))}, ) - return {"requeued": len(retry_rows), "dead_lettered": len(dead_rows)} + ).mappings().first() + return self._job_row(row) if row else None async def due_dispatches(self, *, limit: int = 100) -> list[dict[str, Any]]: bounded_limit = min(max(int(limit), 1), 100) @@ -570,8 +677,8 @@ async def latest_status(self) -> dict[str, Any]: "job_id": None, } mapped_status = { - "created": "running", - "queued": "running", + "created": "queued", + "queued": "queued", "running": "running", "succeeded": "success", "failed": "failed", diff --git a/src/bff/postgres/observability_routes.py b/src/bff/postgres/observability_routes.py index a945584..b3276f5 100644 --- a/src/bff/postgres/observability_routes.py +++ b/src/bff/postgres/observability_routes.py @@ -13,7 +13,7 @@ prefix="/api/admin/observability", tags=["Observability"], dependencies=[ - Depends(require_roles(Role.ADMINISTRATOR, action="observability.read")) + Depends(require_roles(Role.OPERATOR, action="observability.read")) ], ) diff --git a/src/bff/postgres/pipeline_repository.py b/src/bff/postgres/pipeline_repository.py index fbb646e..49c7847 100644 --- a/src/bff/postgres/pipeline_repository.py +++ b/src/bff/postgres/pipeline_repository.py @@ -13,6 +13,50 @@ class PipelineRepository(PostgresRepository): + async def public_source_statuses(self) -> list[dict[str, Any]]: + """Return the intentionally small, credential-free customer status view.""" + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT configuration.source_name, + configuration.enabled, + configuration.last_success_at, + configuration.freshness_sla_hours, + latest.completed_at AS latest_completed_at, + latest.record_count + FROM source_configurations AS configuration + LEFT JOIN LATERAL ( + SELECT completed_at, record_count + FROM source_ingestion_runs + WHERE source_namespace=configuration.source_name + AND status='loaded' + ORDER BY completed_at DESC NULLS LAST, started_at DESC + LIMIT 1 + ) AS latest ON TRUE + ORDER BY configuration.source_name + """ + ) + ) + ).mappings() + return [ + { + "name": str(row["source_name"]), + "enabled": bool(row["enabled"]), + "last_success_at": iso_value( + row["last_success_at"] or row["latest_completed_at"] + ), + "freshness_sla_hours": int(row["freshness_sla_hours"]), + "record_count": ( + int(row["record_count"]) + if row["record_count"] is not None + else None + ), + } + for row in rows + ] + async def synchronize_sources( self, configurations: Iterable[SourceConfiguration] ) -> int: diff --git a/src/bff/postgres/registry_repository.py b/src/bff/postgres/registry_repository.py index 44a16a6..93d8d66 100644 --- a/src/bff/postgres/registry_repository.py +++ b/src/bff/postgres/registry_repository.py @@ -18,6 +18,8 @@ iso_value, number_value, ) +from data.normalization import normalize_organization_name +from preprocessing.enrichment import enrich_organization _SEARCH_SQL = text( @@ -177,6 +179,304 @@ def decode(cls, value: str, signature: str) -> "RegistryPageCursor": class RegistryRepository(PostgresRepository): """Bounded directory filters and lazy record detail for the active dataset.""" + async def link_exact_profile( + self, charity_number: int, *, actor_id: str + ) -> dict[str, Any] | None: + """Create/link one exact cached registry profile and attach observed grants.""" + async with self.sessions() as session, session.begin(): + dataset_version = await self.active_dataset(session) + row = ( + await session.execute( + text( + """ + SELECT registry.registry_id, registry.charity_number, + registry.registered_name, registry.registration_status, + registry.income, registry.expenditure, + registry.address_line_one, registry.address_line_two, + registry.address_line_three, registry.address_line_four, + registry.address_line_five, registry.postcode, + registry.city, registry.administrative_region, + registry.country_code, registry.registered_latitude, + registry.registered_longitude, registry.activity_text, + registry.source_name, registry.source_record_updated_at + FROM charity_registry_organizations AS registry + WHERE registry.dataset_version=:dataset_version + AND registry.is_current_source_record + AND ( + registry.charity_number=:charity_number + OR registry.linked_charity_number=:charity_number + ) + ORDER BY CASE WHEN registry.charity_number=:charity_number + THEN 0 ELSE 1 END, + registry.registry_id + LIMIT 1 + """ + ), + { + "dataset_version": dataset_version, + "charity_number": str(int(charity_number)), + }, + ) + ).mappings().first() + if row is None: + return None + try: + profile_id = int(str(row["charity_number"]).strip()) + except (TypeError, ValueError) as exc: + raise ValueError("The cached registry record has no usable charity number") from exc + + address = ", ".join( + str(row[field]) + for field in ( + "address_line_one", "address_line_two", "address_line_three", + "address_line_four", "address_line_five", "postcode", + ) + if row[field] + ) + source_url = ( + "https://register-of-charities.charitycommission.gov.uk/" + f"en/charity-search/-/charity-details/{profile_id}" + ) + raw_source_data = { + "registered_charity_number": profile_id, + "link": source_url, + "all_details": { + "charity_name": row["registered_name"], + "reg_status": row["registration_status"], + "latest_income": row["income"], + "latest_expenditure": row["expenditure"], + "activities": row["activity_text"], + "address_country": row["country_code"], + }, + "description": row["activity_text"], + "registry_id": row["registry_id"], + } + profile_seed: dict[str, Any] = { + "charity_id": profile_id, + "name": row["registered_name"], + "type": "Charity", + "address": address, + "city": row["city"], + "state": row["administrative_region"], + "country": row["country_code"], + "annual_income": row["income"], + "annual_expenditure": row["expenditure"], + "raw_cc_data": raw_source_data, + } + enrichment = enrich_organization(profile_seed) + source_records = [{ + "source": row["source_name"], + "record_id": row["registry_id"], + "url": source_url, + "record_updated_at": iso_value(row["source_record_updated_at"]), + }] + await session.execute( + text( + """ + INSERT INTO charities ( + dataset_version, charity_id, name, type, address, city, + state, country, latitude, longitude, annual_income, + annual_expenditure, raw_source_data, + programme_areas_source, programme_areas_inferred, + programme_area_scores, programme_area_method, + programme_area_evidence, programme_area_review_required, + geographic_focus_source, geographic_focus_inferred, + headquarters_country, headquarters_region, + geography_method, geography_confidence, geography_evidence, + geography_review_required, enrichment_rule_version, + enrichment_review_reasons, insufficient_source_text, + normalized_name, organization_type, primary_source, + source_names, source_record_id, source_url, source_records, + ingestion_timestamp, transaction_coverage + ) VALUES ( + :dataset_version, :profile_id, :name, 'Charity', :address, + :city, :state, :country, :latitude, :longitude, :income, + :expenditure, CAST(:raw_source_data AS jsonb), + CAST(:programme_areas_source AS jsonb), + CAST(:programme_areas_inferred AS jsonb), + CAST(:programme_area_scores AS jsonb), :programme_area_method, + CAST(:programme_area_evidence AS jsonb), :programme_review, + CAST(:geographic_focus_source AS jsonb), + CAST(:geographic_focus_inferred AS jsonb), + :headquarters_country, :headquarters_region, :geography_method, + :geography_confidence, CAST(:geography_evidence AS jsonb), + :geography_review, :enrichment_rule_version, + CAST(:enrichment_review_reasons AS jsonb), :insufficient_text, + :normalized_name, 'charity', :source_name, + CAST(:source_names AS jsonb), :registry_id, :source_url, + CAST(:source_records AS jsonb), CURRENT_TIMESTAMP, + 'source_without_transactions' + ) + ON CONFLICT (dataset_version, charity_id) DO NOTHING + """ + ), + { + "dataset_version": dataset_version, + "profile_id": profile_id, + "name": str(row["registered_name"]), + "address": address, + "city": row["city"], + "state": row["administrative_region"], + "country": row["country_code"], + "latitude": row["registered_latitude"], + "longitude": row["registered_longitude"], + "income": row["income"], + "expenditure": row["expenditure"], + "raw_source_data": json.dumps(raw_source_data, default=str), + "programme_areas_source": json.dumps(enrichment["programme_areas_source"]), + "programme_areas_inferred": json.dumps(enrichment["programme_areas_inferred"]), + "programme_area_scores": json.dumps(enrichment["programme_area_scores"]), + "programme_area_method": enrichment["programme_area_method"], + "programme_area_evidence": json.dumps(enrichment["programme_area_evidence"]), + "programme_review": bool(enrichment["programme_area_review_required"]), + "geographic_focus_source": json.dumps(enrichment["geographic_focus_source"]), + "geographic_focus_inferred": json.dumps(enrichment["geographic_focus_inferred"]), + "headquarters_country": enrichment["headquarters_country"], + "headquarters_region": enrichment["headquarters_region"], + "geography_method": enrichment["geography_method"], + "geography_confidence": enrichment["geography_confidence"], + "geography_evidence": json.dumps(enrichment["geography_evidence"]), + "geography_review": bool(enrichment["geography_review_required"]), + "enrichment_rule_version": enrichment["enrichment_rule_version"], + "enrichment_review_reasons": json.dumps(enrichment["enrichment_review_reasons"]), + "insufficient_text": bool(enrichment["insufficient_source_text"]), + "normalized_name": normalize_organization_name(row["registered_name"]), + "source_name": row["source_name"], + "source_names": json.dumps([row["source_name"]]), + "registry_id": row["registry_id"], + "source_url": source_url, + "source_records": json.dumps(source_records), + }, + ) + source_identifier = f"GB-CHC-{profile_id}" + linked_grants = ( + await session.execute( + text( + """ + UPDATE grants SET funding_charity_id=:profile_id + WHERE dataset_version=:dataset_version + AND funding_charity_id IS NULL + AND funding_org_source_id=:source_identifier + RETURNING grant_id + """ + ), + { + "dataset_version": dataset_version, + "profile_id": profile_id, + "source_identifier": source_identifier, + }, + ) + ).scalars().all() + linked_facts = ( + await session.execute( + text( + """ + UPDATE grant_source_funder_facts AS facts + SET linked_profile_id=:profile_id + WHERE facts.dataset_version=:dataset_version + AND facts.linked_profile_id IS NULL + AND ( + facts.source_organization_id=:source_identifier + OR EXISTS ( + SELECT 1 FROM grants + WHERE grants.dataset_version=facts.dataset_version + AND grants.grant_id=facts.grant_id + AND grants.funding_charity_id=:profile_id + ) + ) + RETURNING facts.grant_id + """ + ), + { + "dataset_version": dataset_version, + "profile_id": profile_id, + "source_identifier": source_identifier, + }, + ) + ).scalars().all() + observed_grants = int( + await session.scalar( + text( + """ + SELECT COUNT(*) FROM grants + WHERE dataset_version=:dataset_version + AND funding_charity_id=:profile_id + """ + ), + {"dataset_version": dataset_version, "profile_id": profile_id}, + ) + or 0 + ) + await session.execute( + text( + """ + UPDATE charities + SET transaction_coverage=:coverage + WHERE dataset_version=:dataset_version AND charity_id=:profile_id + """ + ), + { + "coverage": ( + "observed_grants_linked" if observed_grants + else "source_without_transactions" + ), + "dataset_version": dataset_version, + "profile_id": profile_id, + }, + ) + await session.execute( + text( + """ + INSERT INTO organization_registry_links ( + dataset_version, registry_id, enriched_organization_id, + match_status, match_method, match_confidence, match_reason, + reviewed_at, created_at, updated_at + ) VALUES ( + :dataset_version, :registry_id, :profile_id, + 'accepted', 'exact_identifier', 1, + :reason, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + ON CONFLICT (dataset_version, registry_id, enriched_organization_id) + DO UPDATE SET match_status='accepted', + match_method='exact_identifier', match_confidence=1, + match_reason=:reason, reviewed_at=CURRENT_TIMESTAMP, + updated_at=CURRENT_TIMESTAMP + """ + ), + { + "dataset_version": dataset_version, + "registry_id": row["registry_id"], + "profile_id": profile_id, + "reason": f"worker-confirmed exact identifier by {actor_id}"[:500], + }, + ) + persisted = await session.scalar( + text( + """ + SELECT COUNT(*) FROM organization_registry_links + WHERE dataset_version=:dataset_version + AND registry_id=:registry_id + AND enriched_organization_id=:profile_id + AND match_status='accepted' + """ + ), + { + "dataset_version": dataset_version, + "registry_id": row["registry_id"], + "profile_id": profile_id, + }, + ) + if int(persisted or 0) != 1: + raise RuntimeError("Registry profile link was not persisted") + return { + "registry_id": str(row["registry_id"]), + "profile_id": profile_id, + "profile_name": str(row["registered_name"]), + "linked_grants": observed_grants, + "newly_linked_grants": len(linked_grants), + "linked_source_facts": len(linked_facts), + } + @staticmethod def _filter_signature(filters: dict[str, Any]) -> str: payload = json.dumps(filters, sort_keys=True, separators=(",", ":"), default=str) diff --git a/src/bff/postgres/routes.py b/src/bff/postgres/routes.py index a63c795..d5ff463 100644 --- a/src/bff/postgres/routes.py +++ b/src/bff/postgres/routes.py @@ -3,11 +3,13 @@ from __future__ import annotations from datetime import date +import hashlib from typing import Any, Optional from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from bff.postgres.analytics_repository import AnalyticsRepository +from bff.postgres.dependencies import reader_sessions, writer_sessions from bff.postgres.funder_repository import SourceFunderRepository from bff.postgres.job_repository import PostgresJobRepository from bff.postgres.organization_repository import OrganizationRepository @@ -43,7 +45,11 @@ def _sessions(request: Request): - return request.app.state.database.sessions() + return reader_sessions(request) + + +def _write_sessions(request: Request): + return writer_sessions(request) def _organizations(request: Request) -> OrganizationRepository: @@ -66,6 +72,14 @@ def _jobs(request: Request) -> PostgresJobRepository: return PostgresJobRepository(_sessions(request)) +def _write_funders(request: Request) -> SourceFunderRepository: + return SourceFunderRepository(_write_sessions(request)) + + +def _write_jobs(request: Request) -> PostgresJobRepository: + return PostgresJobRepository(_write_sessions(request)) + + def _split(value: Optional[str]) -> list[str]: return [item.strip() for item in (value or "").split(",") if item.strip()] @@ -237,7 +251,7 @@ async def registry_detail( async def enrich_registry( payload: SourceFunderEnrichmentRequest, request: Request, - jobs: PostgresJobRepository = Depends(_jobs), + jobs: PostgresJobRepository = Depends(_write_jobs), ): numbers = sorted({int(value) for value in payload.reg_numbers}) if len(numbers) != 1: @@ -247,6 +261,8 @@ async def enrich_registry( {"reg_numbers": numbers, "skip_contact_crawler": payload.skip_contact_crawler}, actor_id=_actor(request), idempotency_key=_idempotency_key(request), + active_dedupe_key=f"registry-enrichment:{numbers[0]}", + max_attempts=1, ) return { "status": job["status"], @@ -395,13 +411,23 @@ async def overview_drilldown( async def enrich_funders( payload: SourceFunderEnrichmentRequest, request: Request, - jobs: PostgresJobRepository = Depends(_jobs), + jobs: PostgresJobRepository = Depends(_write_jobs), ): + targets = sorted( + (str(target.source_funder_key).strip(), int(target.profile_id)) + for target in payload.targets + ) + identity = targets or [ + ("registry", value) for value in sorted(set(payload.reg_numbers)) + ] + dedupe_digest = hashlib.sha256(repr(identity).encode("utf-8")).hexdigest() job = await jobs.enqueue( "source_funder_enrichment", payload.model_dump(), actor_id=_actor(request), idempotency_key=_idempotency_key(request), + active_dedupe_key=f"source-funder-enrichment:{dedupe_digest}", + max_attempts=1, ) return { "status": job["status"], @@ -420,7 +446,7 @@ async def enrich_funders( async def reset_funder( source_funder_key: str, request: Request, - repository: SourceFunderRepository = Depends(_funders), + repository: SourceFunderRepository = Depends(_write_funders), ): result = await repository.reset(source_funder_key, actor_id=_actor(request)) if not result: @@ -436,7 +462,7 @@ async def relink_funder( source_funder_key: str, payload: SourceFunderRelinkRequest, request: Request, - repository: SourceFunderRepository = Depends(_funders), + repository: SourceFunderRepository = Depends(_write_funders), ): try: result = await repository.relink( @@ -456,7 +482,7 @@ async def relink_funder( async def queue_profile_cache( source_funder_key: str, request: Request, - repository: SourceFunderRepository = Depends(_funders), + repository: SourceFunderRepository = Depends(_write_funders), ): result = await repository.queue_profile_cache( source_funder_key, @@ -628,7 +654,7 @@ async def default_charity_score( @router.post( "/{reg_charity_number}/score", response_model=ScoreResponse, - dependencies=[Depends(require_roles(Role.ANALYST, action="score.calculate"))], + dependencies=[Depends(require_roles(Role.OPERATOR, action="score.calculate"))], ) async def charity_score( reg_charity_number: int, diff --git a/src/bff/postgres/scraper_routes.py b/src/bff/postgres/scraper_routes.py new file mode 100644 index 0000000..fd58473 --- /dev/null +++ b/src/bff/postgres/scraper_routes.py @@ -0,0 +1,89 @@ +"""Sanitized customer-visible scraper status without operational controls.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, Request +from pydantic import BaseModel + +from bff.postgres.job_repository import PostgresJobRepository +from bff.postgres.pipeline_repository import PipelineRepository +from bff.postgres.dependencies import reader_sessions +from bff.security import Role, require_roles + + +class PublicSourceStatus(BaseModel): + name: str + enabled: bool + freshness: str + last_success_at: Optional[str] = None + record_count: Optional[int] = None + + +class PublicScraperStatus(BaseModel): + status: str + last_run: Optional[str] = None + last_successful_run: Optional[str] = None + sources: list[PublicSourceStatus] + + +router = APIRouter( + prefix="/api/scraper", + tags=["Scraper Status"], + dependencies=[Depends(require_roles(Role.CUSTOMER, action="scraper.status.read"))], +) + + +def _jobs(request: Request) -> PostgresJobRepository: + return PostgresJobRepository(reader_sessions(request)) + + +def _pipelines(request: Request) -> PipelineRepository: + return PipelineRepository(reader_sessions(request)) + + +def _freshness(last_success_at: Optional[str], sla_hours: int, enabled: bool) -> str: + if not enabled: + return "disabled" + if not last_success_at: + return "never" + try: + observed = datetime.fromisoformat(last_success_at.replace("Z", "+00:00")) + age_hours = (datetime.now(timezone.utc) - observed.astimezone(timezone.utc)).total_seconds() / 3600 + except (TypeError, ValueError): + return "unknown" + return "fresh" if age_hours <= sla_hours else "stale" + + +@router.get("/status", response_model=PublicScraperStatus) +async def scraper_status( + jobs: PostgresJobRepository = Depends(_jobs), + pipelines: PipelineRepository = Depends(_pipelines), +): + latest = await jobs.latest_status() + sources = await pipelines.public_source_statuses() + successful = [source["last_success_at"] for source in sources if source["last_success_at"]] + return { + "status": latest["status"], + "last_run": latest.get("finished_at") or latest.get("started_at"), + "last_successful_run": max(successful) if successful else None, + "sources": [ + { + "name": source["name"], + "enabled": source["enabled"], + "freshness": _freshness( + source["last_success_at"], + source["freshness_sla_hours"], + source["enabled"], + ), + "last_success_at": source["last_success_at"], + "record_count": source["record_count"], + } + for source in sources + ], + } + + +__all__ = ["router"] diff --git a/src/bff/schemas.py b/src/bff/schemas.py index 77ece69..bf4f6b0 100644 --- a/src/bff/schemas.py +++ b/src/bff/schemas.py @@ -674,6 +674,13 @@ class DataSubjectRequestCreate(BaseModel): due_at: Optional[datetime] = None +class SourceFunderEnrichmentTarget(BaseModel): + """One user-confirmed source-funder to active-dataset profile link.""" + + source_funder_key: str = Field(..., min_length=1, max_length=500) + profile_id: int = Field(..., gt=0) + + class SourceFunderEnrichmentRequest(BaseModel): """A deliberately bounded request to enrich observed funders. @@ -684,6 +691,7 @@ class SourceFunderEnrichmentRequest(BaseModel): reg_numbers: List[int] = Field(..., min_length=1, max_length=5) skip_contact_crawler: bool = False + targets: List[SourceFunderEnrichmentTarget] = Field(default=[], max_length=5) class SourceFunderRelinkRequest(BaseModel): diff --git a/src/bff/security.py b/src/bff/security.py index cc56c75..a94b83f 100644 --- a/src/bff/security.py +++ b/src/bff/security.py @@ -16,6 +16,7 @@ import httpx from fastapi import HTTPException, Request, status +from sqlalchemy.exc import SQLAlchemyError from jose import JWTError, jwt from bff.config import SECURITY_SETTINGS, SecuritySettings @@ -23,19 +24,27 @@ class Role(str, Enum): - VIEWER = "viewer" - ANALYST = "analyst" + CUSTOMER = "customer" OPERATOR = "operator" - ADMINISTRATOR = "administrator" + ADMIN = "admin" + + # Compatibility aliases keep the established development/public-readonly + # routes stable while all emitted role values use the three app roles. + VIEWER = "customer" + ANALYST = "customer" + ADMINISTRATOR = "admin" _ROLE_RANK = { - Role.VIEWER: 10, - Role.ANALYST: 20, - Role.OPERATOR: 30, - Role.ADMINISTRATOR: 40, + Role.CUSTOMER: 10, + Role.OPERATOR: 20, + Role.ADMIN: 30, } +APP_ROLES: FrozenSet[Role] = frozenset( + {Role.CUSTOMER, Role.OPERATOR, Role.ADMIN} +) + # This allowlist deliberately contains only the read-only route templates used # by the normal demo UI. Operational, authentication, live-research, cache and @@ -64,6 +73,7 @@ class Role(str, Enum): "/api/charities/grants/summary", "/api/charities/grants/themes", "/api/charities/grants/trends", + "/api/scraper/status", } ) PUBLIC_READONLY_METHOD_ALLOWLIST: FrozenSet[str] = frozenset({"GET", "HEAD"}) @@ -188,7 +198,7 @@ def __init__(self): async def _load_jwks(self, settings: SecuritySettings) -> Mapping[str, Any]: now = time.monotonic() - cache_source = settings.oidc_jwks_json or str(settings.oidc_jwks_url) + cache_source = settings.oidc_jwks_json or _jwks_url(settings) with self._lock: if ( self._jwks @@ -202,7 +212,7 @@ async def _load_jwks(self, settings: SecuritySettings) -> Mapping[str, Any]: else: try: async with httpx.AsyncClient(timeout=5.0, follow_redirects=False) as client: - response = await client.get(str(settings.oidc_jwks_url)) + response = await client.get(_jwks_url(settings)) response.raise_for_status() if len(response.content) > 256_000: raise ValueError("OIDC JWKS response exceeds the configured safety bound") @@ -239,6 +249,25 @@ async def decode(self, token: str, settings: SecuritySettings) -> Mapping[str, A if len(matching_keys) != 1: raise _invalid_token() try: + if settings.auth_mode == "cognito_rbac": + claims = jwt.decode( + token, + matching_keys[0], + algorithms=list(settings.oidc_algorithms), + issuer=_issuer(settings), + options={ + "verify_aud": False, + "require_exp": True, + "require_iat": True, + "require_sub": True, + }, + ) + if ( + claims.get("token_use") != "access" + or claims.get("client_id") != settings.cognito_client_id + ): + raise _invalid_token() + return claims return jwt.decode( token, matching_keys[0], @@ -247,6 +276,8 @@ async def decode(self, token: str, settings: SecuritySettings) -> Mapping[str, A issuer=settings.oidc_issuer, options={"require_exp": True, "require_iat": True, "require_sub": True}, ) + except HTTPException: + raise except JWTError as exc: logger.warning("OIDC token validation failed: %s", exc.__class__.__name__) raise _invalid_token() from exc @@ -255,6 +286,21 @@ async def decode(self, token: str, settings: SecuritySettings) -> Mapping[str, A _oidc_verifier = OIDCVerifier() +def _issuer(settings: SecuritySettings) -> str: + if settings.auth_mode == "cognito_rbac": + return ( + f"https://cognito-idp.{settings.cognito_region}.amazonaws.com/" + f"{settings.cognito_user_pool_id}" + ) + return str(settings.oidc_issuer) + + +def _jwks_url(settings: SecuritySettings) -> str: + if settings.auth_mode == "cognito_rbac": + return f"{_issuer(settings)}/.well-known/jwks.json" + return str(settings.oidc_jwks_url) + + def _invalid_token() -> HTTPException: return HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -294,7 +340,7 @@ def _public_readonly_principal( return None return Principal( actor_id="public-readonly-demo", - roles=frozenset({Role.VIEWER}), + roles=frozenset({Role.CUSTOMER}), claims={"sub": "public-readonly-demo", "auth_mode": "public_readonly"}, ) @@ -308,7 +354,11 @@ def _roles_from_claims(claims: Mapping[str, Any], claim_name: str) -> FrozenSet[ else: candidates = [] normalized = set() - aliases = {"admin": Role.ADMINISTRATOR, "administrator": Role.ADMINISTRATOR} + aliases = { + "viewer": Role.CUSTOMER, + "analyst": Role.CUSTOMER, + "administrator": Role.ADMIN, + } for candidate in candidates: value = candidate.strip().lower() try: @@ -319,9 +369,29 @@ def _roles_from_claims(claims: Mapping[str, Any], claim_name: str) -> FrozenSet[ return frozenset(normalized) +def _cognito_roles_from_claims(claims: Mapping[str, Any]) -> FrozenSet[Role]: + raw_groups = claims.get("cognito:groups", []) + if not isinstance(raw_groups, (list, tuple, set)): + raw_groups = [] + roles: set[Role] = set() + for group in raw_groups: + try: + role = Role(str(group).strip().lower()) + except ValueError: + continue + if role in APP_ROLES: + roles.add(role) + if len(roles) != 1: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Exactly one application role group is required.", + ) + return frozenset(roles) + + def create_development_access_token( subject: str, - roles: Iterable[Role] = (Role.ADMINISTRATOR,), + roles: Iterable[Role] = (Role.ADMIN,), *, settings: SecuritySettings = SECURITY_SETTINGS, expires_delta: Optional[timedelta] = None, @@ -359,7 +429,7 @@ async def _decode_token(token: str, request: Request, settings: SecuritySettings ) except JWTError as exc: raise _invalid_token() from exc - if settings.auth_mode == "oidc": + if settings.auth_mode in {"oidc", "cognito_rbac"}: return await _oidc_verifier.decode(token, settings) raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -416,9 +486,14 @@ async def authenticate_request(request: Request) -> Principal: subject = claims.get("sub") if not isinstance(subject, str) or not subject.strip(): raise _invalid_token() + roles = ( + _cognito_roles_from_claims(claims) + if settings.auth_mode == "cognito_rbac" + else _roles_from_claims(claims, settings.oidc_role_claim) + ) principal = Principal( actor_id=subject.strip(), - roles=_roles_from_claims(claims, settings.oidc_role_claim), + roles=roles, claims=claims, ) retry_after = _limiter_for(request, settings).check(f"actor:{principal.actor_id}") @@ -457,6 +532,15 @@ async def reserve_idempotency(request: Request, principal: Principal, action: st status_code=status.HTTP_409_CONFLICT, detail=str(exc), ) from exc + except Exception as exc: + from bff.database import WriterDatabaseUnavailable + + if not isinstance(exc, (WriterDatabaseUnavailable, SQLAlchemyError)): + raise + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="The mutation database path is temporarily unavailable.", + ) from exc request.state.idempotency_record_key = record_key @@ -479,6 +563,22 @@ async def dependency(request: Request) -> Principal: return dependency +def require_authenticated_user(*, action: str = "authenticated.read"): + return require_roles(Role.CUSTOMER, action=action) + + +def require_role(role: Role, *, action: str = "authenticated.read", idempotent: bool = False): + return require_roles(role, action=action, idempotent=idempotent) + + +def require_any_role( + *roles: Role, + action: str = "authenticated.read", + idempotent: bool = False, +): + return require_roles(*roles, action=action, idempotent=idempotent) + + async def enforce_login_rate_limit(request: Request) -> None: settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) retry_after = _limiter_for(request, settings).check(f"login:{_request_host(request)}") diff --git a/src/bff/user_management.py b/src/bff/user_management.py new file mode 100644 index 0000000..88b8b53 --- /dev/null +++ b/src/bff/user_management.py @@ -0,0 +1,365 @@ +"""Fail-closed Cognito user administration for the three application roles.""" + +from __future__ import annotations + +from typing import Any, cast, Literal, Mapping, Optional + +from fastapi import APIRouter, Depends, HTTPException, Path, Query, Request, status +from pydantic import BaseModel, Field, field_validator + +from bff.config import SECURITY_SETTINGS +from bff.security import APP_ROLES, Principal, Role, require_roles + + +CognitoRole = Literal["customer", "operator", "admin"] +APP_GROUP_NAMES = frozenset(role.value for role in APP_ROLES) + + +class UserCreate(BaseModel): + email: str = Field(min_length=3, max_length=320) + role: str = Field(min_length=1, max_length=32) + + @field_validator("email") + @classmethod + def validate_email(cls, value: str) -> str: + normalized = value.strip().lower() + if normalized.count("@") != 1 or normalized.startswith("@") or normalized.endswith("@"): + raise ValueError("A valid email address is required") + return normalized + + +class UserRoleUpdate(BaseModel): + role: str = Field(min_length=1, max_length=32) + + +class UserRecord(BaseModel): + id: str + email: Optional[str] = None + enabled: bool + status: str + role: Optional[CognitoRole] = None + + +class UserPage(BaseModel): + users: list[UserRecord] + next_token: Optional[str] = None + + +router = APIRouter(prefix="/api/admin/users", tags=["User Management"]) + + +def _settings(request: Request): + settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) + if settings.auth_mode != "cognito_rbac" or not settings.cognito_user_pool_id: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + return settings + + +def _client(request: Request): + existing = getattr(request.app.state, "cognito_client", None) + if existing is not None: + return existing + settings = _settings(request) + try: + import boto3 + except ImportError as exc: # pragma: no cover - deployment dependency contract + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Identity administration is temporarily unavailable.", + ) from exc + client = boto3.client("cognito-idp", region_name=settings.cognito_region) + request.app.state.cognito_client = client + return client + + +def _pool_id(request: Request) -> str: + return str(_settings(request).cognito_user_pool_id) + + +def _error_code(exc: Exception) -> str: + response = getattr(exc, "response", {}) + if isinstance(response, Mapping): + error = response.get("Error", {}) + if isinstance(error, Mapping): + return str(error.get("Code") or "") + return "" + + +def _raise_safe_cognito_error(exc: Exception) -> None: + code = _error_code(exc) + if code in {"UsernameExistsException", "AliasExistsException"}: + raise HTTPException(status_code=409, detail="A user with this email already exists.") from exc + if code == "UserNotFoundException": + raise HTTPException(status_code=404, detail="User not found.") from exc + if code in {"InvalidParameterException", "InvalidPasswordException"}: + raise HTTPException(status_code=400, detail="The identity provider rejected the request.") from exc + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Identity provider operation failed.", + ) from exc + + +def _attribute_map(user: Mapping[str, Any]) -> dict[str, str]: + result: dict[str, str] = {} + for item in user.get("Attributes", []) or user.get("UserAttributes", []): + if isinstance(item, Mapping) and isinstance(item.get("Name"), str): + result[str(item["Name"])] = str(item.get("Value") or "") + return result + + +def _validated_role(value: str) -> CognitoRole: + normalized = value.strip().lower() + if normalized not in APP_GROUP_NAMES: + raise HTTPException(status_code=400, detail="Role must be customer, operator or admin.") + return cast(CognitoRole, normalized) + + +def _groups_for_user(client: Any, pool_id: str, username: str) -> set[str]: + groups: set[str] = set() + next_token: Optional[str] = None + while True: + arguments: dict[str, Any] = { + "UserPoolId": pool_id, + "Username": username, + "Limit": 60, + } + if next_token: + arguments["NextToken"] = next_token + response = client.admin_list_groups_for_user(**arguments) + groups.update( + str(group.get("GroupName")) + for group in response.get("Groups", []) + if isinstance(group, Mapping) and group.get("GroupName") in APP_GROUP_NAMES + ) + next_token = response.get("NextToken") + if not next_token: + return groups + + +def _record(client: Any, pool_id: str, user: Mapping[str, Any]) -> UserRecord: + username = str(user.get("Username") or "") + groups = _groups_for_user(client, pool_id, username) + role: Optional[CognitoRole] = None + if len(groups) == 1: + role = cast(CognitoRole, next(iter(groups))) + attributes = _attribute_map(user) + return UserRecord( + id=username, + email=attributes.get("email") or None, + enabled=bool(user.get("Enabled", False)), + status=str(user.get("UserStatus") or "UNKNOWN"), + role=role, + ) + + +def _get_user(client: Any, pool_id: str, username: str) -> Mapping[str, Any]: + try: + return client.admin_get_user(UserPoolId=pool_id, Username=username) + except Exception as exc: # SDK exceptions are intentionally translated centrally. + _raise_safe_cognito_error(exc) + raise AssertionError("unreachable") + + +def _target_is_current_user(target: Mapping[str, Any], principal: Principal) -> bool: + target_sub = _attribute_map(target).get("sub") + return bool(target_sub and target_sub == principal.actor_id) + + +def _active_admin_usernames(client: Any, pool_id: str) -> list[str]: + usernames: list[str] = [] + next_token: Optional[str] = None + while True: + arguments: dict[str, Any] = { + "UserPoolId": pool_id, + "GroupName": Role.ADMIN.value, + "Limit": 60, + } + if next_token: + arguments["NextToken"] = next_token + response = client.list_users_in_group(**arguments) + for listed in response.get("Users", []): + username = str(listed.get("Username") or "") + if not username: + continue + current = _get_user(client, pool_id, username) + if bool(current.get("Enabled", False)): + usernames.append(username) + next_token = response.get("NextToken") + if not next_token: + return usernames + + +def _protect_admin_change( + client: Any, + pool_id: str, + username: str, + target: Mapping[str, Any], + principal: Principal, +) -> None: + if _target_is_current_user(target, principal): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Administrators cannot disable or downgrade their own account.", + ) + active_admins = _active_admin_usernames(client, pool_id) + if username in active_admins and len(active_admins) <= 1: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="The last active administrator cannot be disabled or downgraded.", + ) + + +@router.get("", response_model=UserPage) +async def list_users( + request: Request, + page_size: int = Query(default=25, ge=1, le=50), + next_token: Optional[str] = Query(default=None, min_length=1, max_length=4096), + _: Principal = Depends(require_roles(Role.ADMIN, action="users.list")), +): + client = _client(request) + pool_id = _pool_id(request) + arguments: dict[str, Any] = {"UserPoolId": pool_id, "Limit": page_size} + if next_token: + arguments["PaginationToken"] = next_token + try: + response = client.list_users(**arguments) + users = [_record(client, pool_id, user) for user in response.get("Users", [])] + except HTTPException: + raise + except Exception as exc: + _raise_safe_cognito_error(exc) + return UserPage(users=users, next_token=response.get("PaginationToken")) + + +@router.post("", response_model=UserRecord, status_code=status.HTTP_201_CREATED) +async def create_user( + payload: UserCreate, + request: Request, + _: Principal = Depends( + require_roles(Role.ADMIN, action="users.create", idempotent=True) + ), +): + client = _client(request) + pool_id = _pool_id(request) + role = _validated_role(payload.role) + try: + response = client.admin_create_user( + UserPoolId=pool_id, + Username=payload.email, + UserAttributes=[{"Name": "email", "Value": payload.email}], + DesiredDeliveryMediums=["EMAIL"], + ) + username = str(response["User"]["Username"]) + client.admin_add_user_to_group( + UserPoolId=pool_id, + Username=username, + GroupName=role, + ) + user = _get_user(client, pool_id, username) + return _record(client, pool_id, user) + except HTTPException: + raise + except Exception as exc: + _raise_safe_cognito_error(exc) + + +@router.patch("/{user_id}/role", response_model=UserRecord) +async def update_user_role( + payload: UserRoleUpdate, + request: Request, + user_id: str = Path(min_length=1, max_length=128), + principal: Principal = Depends( + require_roles(Role.ADMIN, action="users.role.update", idempotent=True) + ), +): + client = _client(request) + pool_id = _pool_id(request) + role = _validated_role(payload.role) + target = _get_user(client, pool_id, user_id) + groups = _groups_for_user(client, pool_id, user_id) + if Role.ADMIN.value in groups and role != Role.ADMIN.value: + _protect_admin_change(client, pool_id, user_id, target, principal) + try: + if role not in groups: + client.admin_add_user_to_group( + UserPoolId=pool_id, + Username=user_id, + GroupName=role, + ) + for group in sorted(groups - {role}): + client.admin_remove_user_from_group( + UserPoolId=pool_id, + Username=user_id, + GroupName=group, + ) + return _record(client, pool_id, _get_user(client, pool_id, user_id)) + except HTTPException: + raise + except Exception as exc: + _raise_safe_cognito_error(exc) + + +@router.post("/{user_id}/disable", response_model=UserRecord) +async def disable_user( + request: Request, + user_id: str = Path(min_length=1, max_length=128), + principal: Principal = Depends( + require_roles(Role.ADMIN, action="users.disable", idempotent=True) + ), +): + client = _client(request) + pool_id = _pool_id(request) + target = _get_user(client, pool_id, user_id) + groups = _groups_for_user(client, pool_id, user_id) + if _target_is_current_user(target, principal): + _protect_admin_change(client, pool_id, user_id, target, principal) + if Role.ADMIN.value in groups: + _protect_admin_change(client, pool_id, user_id, target, principal) + try: + client.admin_disable_user(UserPoolId=pool_id, Username=user_id) + return _record(client, pool_id, _get_user(client, pool_id, user_id)) + except HTTPException: + raise + except Exception as exc: + _raise_safe_cognito_error(exc) + + +@router.post("/{user_id}/enable", response_model=UserRecord) +async def enable_user( + request: Request, + user_id: str = Path(min_length=1, max_length=128), + _: Principal = Depends( + require_roles(Role.ADMIN, action="users.enable", idempotent=True) + ), +): + client = _client(request) + pool_id = _pool_id(request) + _get_user(client, pool_id, user_id) + try: + client.admin_enable_user(UserPoolId=pool_id, Username=user_id) + return _record(client, pool_id, _get_user(client, pool_id, user_id)) + except HTTPException: + raise + except Exception as exc: + _raise_safe_cognito_error(exc) + + +@router.post("/{user_id}/reset-password") +async def reset_user_password( + request: Request, + user_id: str = Path(min_length=1, max_length=128), + _: Principal = Depends( + require_roles(Role.ADMIN, action="users.password.reset", idempotent=True) + ), +): + client = _client(request) + pool_id = _pool_id(request) + _get_user(client, pool_id, user_id) + try: + client.admin_reset_user_password(UserPoolId=pool_id, Username=user_id) + except Exception as exc: + _raise_safe_cognito_error(exc) + return {"message": "Password reset initiated."} + + +__all__ = ["router"] diff --git a/src/data/normalization.py b/src/data/normalization.py new file mode 100644 index 0000000..15645e1 --- /dev/null +++ b/src/data/normalization.py @@ -0,0 +1,30 @@ +"""Pure organization identity normalization shared by SQLite and PostgreSQL.""" + +from __future__ import annotations + +import re +from typing import Any +import unicodedata + + +def normalize_organization_name(value: Any) -> str: + """Return a conservative search-safe organization-name representation.""" + normalized = unicodedata.normalize("NFKC", str(value or "")).casefold() + normalized = normalized.replace("&", " and ") + normalized = re.sub(r"[\u2010-\u2015/_.,;:()\[\]{}'\"`]+", " ", normalized) + normalized = re.sub(r"\s+", " ", normalized).strip() + for suffix in ( + "charitable incorporated organisation", + "charitable incorporated organization", + "community interest company", + "limited liability partnership", + "company limited by guarantee", + "limited", + "ltd", + "plc", + "cio", + "cic", + ): + if normalized.endswith(f" {suffix}"): + return normalized[: -len(suffix)].strip() + return normalized diff --git a/src/data/registry.py b/src/data/registry.py index c19a653..0522875 100644 --- a/src/data/registry.py +++ b/src/data/registry.py @@ -12,11 +12,12 @@ import os import re import sqlite3 -import unicodedata from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple +from data.normalization import normalize_organization_name + REGISTRY_TABLE = "charity_registry_organizations" REGISTRY_LINK_TABLE = "organization_registry_links" @@ -29,36 +30,6 @@ def utc_now() -> str: return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") -def normalize_organization_name(value: Any) -> str: - """Return a conservative, search-safe organization-name representation. - - Legal suffixes are removed only when they are trailing standalone tokens. The - resulting value is suitable for search and conservative exact-name matching; - it is never used by itself as authority for a grant or score relationship. - """ - text = unicodedata.normalize("NFKC", str(value or "")).casefold() - text = text.replace("&", " and ") - text = re.sub(r"[\u2010-\u2015/_.,;:()\[\]{}'\"`]+", " ", text) - text = re.sub(r"\s+", " ", text).strip() - suffixes = ( - "charitable incorporated organisation", - "charitable incorporated organization", - "community interest company", - "limited liability partnership", - "company limited by guarantee", - "limited", - "ltd", - "plc", - "cio", - "cic", - ) - for suffix in suffixes: - if text.endswith(f" {suffix}"): - text = text[: -len(suffix)].strip() - break - return text - - def _fts_available(conn: sqlite3.Connection) -> bool: return bool( conn.execute( diff --git a/src/migration/aws_entrypoint.py b/src/migration/aws_entrypoint.py index 02e3934..31aa686 100644 --- a/src/migration/aws_entrypoint.py +++ b/src/migration/aws_entrypoint.py @@ -22,6 +22,10 @@ import asyncpg from bff.database import DatabaseManager, DatabaseSettings +from migration.database_access import ( + bootstrap_runtime_configuration, + configure_reader_role, +) from migration.release_gate import release_state from migration.sqlite_to_postgres import migrate, source_preflight @@ -92,69 +96,16 @@ async def _configure_application_role( connection: asyncpg.Connection, environment: Mapping[str, str], ) -> None: - application_user = _identifier( - _required(environment, "DATABASE_APP_USER"), "DATABASE_APP_USER" - ) - admin_user = _identifier( - _required(environment, "DATABASE_ADMIN_USER"), "DATABASE_ADMIN_USER" - ) - database_name = _identifier( - _required(environment, "DATABASE_NAME"), "DATABASE_NAME" - ) - application_password = _required(environment, "DATABASE_APP_PASSWORD") - role = _quoted_identifier(application_user) - password = _quoted_literal(application_password) - exists = await connection.fetchval( - "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname=$1)", - application_user, - ) - if exists: - await connection.execute( - f"ALTER ROLE {role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE " - f"NOINHERIT NOREPLICATION PASSWORD {password}" - ) - else: - await connection.execute( - f"CREATE ROLE {role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE " - f"NOINHERIT NOREPLICATION PASSWORD {password}" - ) - await connection.execute(f"REVOKE ALL ON DATABASE {_quoted_identifier(database_name)} FROM PUBLIC") - await connection.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC") - await connection.execute( - f"GRANT CONNECT ON DATABASE {_quoted_identifier(database_name)} TO {role}" - ) - await connection.execute(f"GRANT USAGE ON SCHEMA public TO {role}") - await connection.execute( - f"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM {role}" - ) - await connection.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA public TO {role}") - await connection.execute( - f"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM {role}" - ) - await connection.execute("REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC") - await connection.execute(f"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM {role}") - owner = _quoted_identifier(admin_user) - await connection.execute( - f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " - f"REVOKE ALL ON TABLES FROM {role}" - ) - await connection.execute( - f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " - f"GRANT SELECT ON TABLES TO {role}" - ) - await connection.execute( - f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " - f"REVOKE ALL ON SEQUENCES FROM {role}" - ) - await connection.execute( - f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " - "REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC" - ) - await connection.execute( - f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " - f"REVOKE ALL ON FUNCTIONS FROM {role}" + reader_environment = dict(environment) + reader_environment.update( + { + "DATABASE_READER_USER": _required(environment, "DATABASE_APP_USER"), + "DATABASE_READER_PASSWORD": _required( + environment, "DATABASE_APP_PASSWORD" + ), + } ) - await connection.execute(f"ALTER ROLE {role} SET default_transaction_read_only = on") + await configure_reader_role(connection, reader_environment) async def _verify_application_role( @@ -265,6 +216,7 @@ async def run(environment: Mapping[str, str] | None = None) -> dict[str, object] output_directory, remote_postgres=True, ) + configuration_bootstrap = await bootstrap_runtime_configuration(admin) await _configure_application_role(admin, task_environment) application_role = await _verify_application_role(task_environment) os.environ.update( @@ -295,6 +247,7 @@ async def run(environment: Mapping[str, str] | None = None) -> dict[str, object] "dataset_version": report["dataset_version"], "activation_status": report["activation_status"], "application_role": application_role, + "configuration_bootstrap": configuration_bootstrap, "release_gate_ready": True, } output_directory.mkdir(parents=True, exist_ok=True, mode=0o700) diff --git a/src/migration/database_access.py b/src/migration/database_access.py new file mode 100644 index 0000000..ea96bfb --- /dev/null +++ b/src/migration/database_access.py @@ -0,0 +1,1333 @@ +"""Least-privilege PostgreSQL runtime role provisioning and verification. + +This module is invoked only by an explicit migration/release task. The +long-running application imports the resulting reader/writer credentials but +never receives the master credential used here. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import re +from typing import Any, Iterable, Mapping +import uuid + +import asyncpg + +from governance.retention import GovernanceConfiguration, load_governance_configuration +from pipelines.durable import SourceConfiguration, load_source_configurations + + +DATABASE_ACCESS_LOCK_ID = 2_083_370_803_871 +IDENTIFIER_PATTERN = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") + +# Every relation read by the PostgreSQL API, readiness contract, and sanitized +# operational read surface is explicit. New tables fail closed until reviewed. +READER_TABLES = ( + "alembic_version", + "analytics_country_aggregates", + "analytics_country_connections", + "analytics_country_funder_rankings", + "analytics_entity_rankings", + "analytics_filter_values", + "analytics_funder_relationships", + "analytics_period_aggregates", + "analytics_programme_aggregates", + "analytics_scope_totals", + "charities", + "charity_registry_organizations", + "data_quality_issues", + "data_holds", + "dataset_versions", + "export_jobs", + "grant_beneficiary_countries", + "grant_overview_facts", + "grant_programme_categories", + "grant_source_funder_facts", + "grants", + "job_dispatch_outbox", + "job_events", + "job_runs", + "materialization_versions", + "migration_runs", + "organization_registry_links", + "retention_policies", + "source_configurations", + "source_funder_link_overrides", + "source_funder_profile_cache", + "source_ingestion_runs", +) + +# Writer SELECT is limited to state needed by an explicit mutation or worker. +WRITER_READ_TABLES = ( + "charities", + "data_holds", + "dataset_versions", + "grant_source_funder_facts", + "idempotency_records", + "job_dispatch_outbox", + "job_events", + "job_runs", + "source_configurations", + "source_funder_link_overrides", + "source_funder_profile_cache", + "source_ingestion_runs", + "worker_heartbeats", +) + +# No GRANT ALL and no blanket DML. These privileges map one-to-one to audited +# repository functions and background worker transitions. +WRITER_TABLE_PRIVILEGES: Mapping[str, tuple[str, ...]] = { + "data_holds": ("INSERT", "UPDATE"), + "data_subject_requests": ("INSERT",), + "deletion_manifests": ("INSERT",), + "idempotency_records": ("INSERT", "UPDATE", "DELETE"), + "ingestion_run_manifests": ("INSERT",), + "job_dispatch_outbox": ("INSERT", "UPDATE"), + "job_events": ("INSERT",), + "job_runs": ("INSERT", "UPDATE"), + "restore_verifications": ("INSERT",), + "retention_actions": ("INSERT",), + "source_funder_link_overrides": ("INSERT", "UPDATE"), + "source_funder_profile_cache": ("INSERT", "UPDATE", "DELETE"), + "source_ingestion_runs": ("INSERT", "UPDATE"), + "storage_objects": ("INSERT",), + "worker_heartbeats": ("INSERT", "UPDATE"), +} +WRITER_COLUMN_PRIVILEGES: Mapping[str, Mapping[str, tuple[str, ...]]] = { + "source_configurations": {"UPDATE": ("enabled", "updated_at")}, +} +WRITER_SEQUENCES: tuple[str, ...] = () + +# The long-running worker gets a separate publisher principal. It can load and +# atomically activate versioned datasets, but it cannot create schema objects, +# roles or databases and is never shared with browser-facing API mutations. +PIPELINE_VERSIONED_TABLES = ( + "charities", + "charity_registry_organizations", + "grants", + "grant_beneficiary_countries", + "grant_beneficiary_terms", + "grant_programme_categories", + "grant_overview_facts", + "grant_source_funder_facts", + "organization_registry_links", + "source_funder_profile_cache", +) +PIPELINE_ANALYTICS_TABLES = ( + "analytics_scope_totals", + "analytics_country_aggregates", + "analytics_country_connections", + "analytics_period_aggregates", + "analytics_programme_aggregates", + "analytics_entity_rankings", + "analytics_country_funder_rankings", + "analytics_funder_relationships", + "analytics_filter_values", + "materialization_versions", +) +PIPELINE_CONTROL_TABLES = ( + "dataset_versions", + "migration_runs", + "data_quality_issues", +) +PIPELINE_GLOBAL_TABLES = ( + "source_funder_link_overrides", + "exchange_rates", +) +PIPELINE_READ_TABLES = ( + "alembic_version", + *PIPELINE_VERSIONED_TABLES, + *PIPELINE_ANALYTICS_TABLES, + *PIPELINE_CONTROL_TABLES, + *PIPELINE_GLOBAL_TABLES, +) +PIPELINE_TABLE_PRIVILEGES: Mapping[str, tuple[str, ...]] = { + table: ("INSERT", "UPDATE", "DELETE") + for table in ( + *PIPELINE_VERSIONED_TABLES, + *PIPELINE_ANALYTICS_TABLES, + *PIPELINE_CONTROL_TABLES, + *PIPELINE_GLOBAL_TABLES, + ) +} +PIPELINE_FUNCTIONS = ("refresh_analytics_materializations(TEXT)",) +RUNTIME_ROLE_MEMBERSHIP_ALLOWLIST: tuple[str, ...] = () +PUBLIC_FUNCTION_EXTENSION_ALLOWLIST = ("pg_trgm",) + +SOURCE_CONFIGURATION_INTEGRITY_FIELDS = ( + "source_name", + "source_owner", + "technical_owner", + "legal_status", + "licence_status", + "terms_url", + "rate_limit_per_minute", + "user_agent", + "freshness_sla_hours", + "schedule_expression", + "enabled", + "governance_blocked", + "last_success_at", + "watermark", + "classification", + "retention_class", + "schema_version", + "retry_limit", + "timeout_seconds", + "maximum_pages", + "maximum_records", + "configuration_checksum", +) +RETENTION_POLICY_INTEGRITY_FIELDS = ( + "retention_class", + "classification", + "policy_status", + "archive_after_days", + "delete_after_days", + "export_expire_after_days", + "destructive_deletion_enabled", + "production_approved", + "approval_reference", + "configuration_checksum", +) + + +class DatabaseAccessConfigurationError(RuntimeError): + """Raised before privileged SQL when prerequisite input is unsafe.""" + + +def _required(environment: Mapping[str, str], name: str) -> str: + value = environment.get(name, "").strip() + if not value: + raise DatabaseAccessConfigurationError(f"{name} is required") + return value + + +def identifier(value: str, name: str) -> str: + if not IDENTIFIER_PATTERN.fullmatch(value): + raise DatabaseAccessConfigurationError(f"{name} is not a safe PostgreSQL identifier") + return value + + +def _quoted_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _quoted_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +async def connect_admin(environment: Mapping[str, str]) -> asyncpg.Connection: + return await asyncpg.connect( + host=_required(environment, "DATABASE_HOST"), + port=int(environment.get("DATABASE_PORT", "5432")), + user=_required(environment, "DATABASE_ADMIN_USER"), + password=_required(environment, "DATABASE_ADMIN_PASSWORD"), + database=_required(environment, "DATABASE_NAME"), + ssl=environment.get("DATABASE_SSL_MODE", "disable").strip().lower(), + command_timeout=None, + ) + + +async def _configure_login_role( + connection: asyncpg.Connection, + *, + username: str, + password: str, + default_read_only: bool, +) -> None: + role = _quoted_identifier(username) + password_literal = _quoted_literal(password) + state = await runtime_role_state(connection, username) + if state is not None: + _assert_runtime_role_safe(state, username=username) + # SUPERUSER/NOSUPERUSER, REPLICATION and BYPASSRLS are deliberately + # absent here. RDS release principals are not PostgreSQL superusers and + # cannot alter those attributes, even to their already-safe values. + await connection.execute( + f"ALTER ROLE {role} LOGIN NOINHERIT PASSWORD {password_literal}" + ) + else: + await connection.execute( + f"CREATE ROLE {role} LOGIN NOCREATEDB NOCREATEROLE NOINHERIT " + f"NOREPLICATION NOBYPASSRLS PASSWORD {password_literal}" + ) + _assert_runtime_role_safe( + await _required_runtime_role_state(connection, username), + username=username, + require_login_posture=True, + ) + # Read pg_roles again immediately before the second ALTER ROLE. This also + # makes an unexpected concurrent privilege or membership change fail closed. + _assert_runtime_role_safe( + await _required_runtime_role_state(connection, username), + username=username, + require_login_posture=True, + ) + await connection.execute( + f"ALTER ROLE {role} SET default_transaction_read_only = " + f"{'on' if default_read_only else 'off'}" + ) + _assert_runtime_role_safe( + await _required_runtime_role_state(connection, username), + username=username, + require_login_posture=True, + ) + + +async def runtime_role_state( + connection: asyncpg.Connection, username: str +) -> dict[str, Any] | None: + row = await connection.fetchrow( + """ + SELECT rolname, rolsuper, rolcreaterole, rolcreatedb, rolreplication, + rolbypassrls, rolinherit, rolcanlogin + FROM pg_roles + WHERE rolname=$1 + """, + username, + ) + if row is None: + return None + memberships = await connection.fetch( + """ + SELECT parent.rolname AS parent_role + FROM pg_auth_members AS membership + JOIN pg_roles AS parent ON parent.oid=membership.roleid + JOIN pg_roles AS member ON member.oid=membership.member + WHERE member.rolname=$1 + ORDER BY parent.rolname + """, + username, + ) + state = dict(row) + state["memberships"] = [str(item["parent_role"]) for item in memberships] + return state + + +async def _required_runtime_role_state( + connection: asyncpg.Connection, username: str +) -> dict[str, Any]: + state = await runtime_role_state(connection, username) + if state is None: + raise DatabaseAccessConfigurationError( + f"runtime role {username} disappeared during provisioning" + ) + return state + + +def _assert_runtime_role_safe( + state: Mapping[str, Any], *, username: str, require_login_posture: bool = False +) -> None: + dangerous = ( + "rolsuper", + "rolcreaterole", + "rolcreatedb", + "rolreplication", + "rolbypassrls", + ) + enabled = [attribute for attribute in dangerous if bool(state[attribute])] + if enabled: + raise DatabaseAccessConfigurationError( + f"runtime role {username} has prohibited attributes: {', '.join(enabled)}" + ) + memberships = set(str(item) for item in state.get("memberships", ())) + unexpected = memberships.difference(RUNTIME_ROLE_MEMBERSHIP_ALLOWLIST) + if unexpected: + raise DatabaseAccessConfigurationError( + f"runtime role {username} has unexpected memberships" + ) + if require_login_posture and not bool(state["rolcanlogin"]): + raise DatabaseAccessConfigurationError( + f"runtime role {username} is not a login role" + ) + if require_login_posture and bool(state["rolinherit"]): + raise DatabaseAccessConfigurationError( + f"runtime role {username} unexpectedly inherits privileges" + ) + + +async def _reset_runtime_privileges( + connection: asyncpg.Connection, + *, + username: str, + database_name: str, + owner_username: str, +) -> None: + role = _quoted_identifier(username) + database = _quoted_identifier(database_name) + owner = _quoted_identifier(owner_username) + await connection.execute(f"REVOKE ALL ON DATABASE {database} FROM {role}") + await connection.execute(f"GRANT CONNECT ON DATABASE {database} TO {role}") + await connection.execute(f"REVOKE ALL ON SCHEMA public FROM {role}") + await connection.execute(f"GRANT USAGE ON SCHEMA public TO {role}") + await connection.execute(f"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM {role}") + column_grants = await connection.fetch( + """ + SELECT table_name, privilege_type, + array_agg(column_name ORDER BY column_name) AS column_names + FROM information_schema.column_privileges + WHERE table_schema='public' AND grantee=$1 + GROUP BY table_name, privilege_type + ORDER BY table_name, privilege_type + """, + username, + ) + for grant in column_grants: + columns = ", ".join( + _quoted_identifier(str(column)) for column in grant["column_names"] + ) + await connection.execute( + f"REVOKE {grant['privilege_type']} ({columns}) ON TABLE " + f"{_quoted_identifier(str(grant['table_name']))} FROM {role}" + ) + await connection.execute(f"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM {role}") + await connection.execute(f"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM {role}") + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + f"REVOKE ALL ON TABLES FROM {role}" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + f"REVOKE ALL ON SEQUENCES FROM {role}" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + f"REVOKE ALL ON FUNCTIONS FROM {role}" + ) + + +async def configure_reader_role( + connection: asyncpg.Connection, + environment: Mapping[str, str], +) -> None: + username = identifier( + _required(environment, "DATABASE_READER_USER"), "DATABASE_READER_USER" + ) + password = _required(environment, "DATABASE_READER_PASSWORD") + owner = identifier( + _required(environment, "DATABASE_ADMIN_USER"), "DATABASE_ADMIN_USER" + ) + database = identifier(_required(environment, "DATABASE_NAME"), "DATABASE_NAME") + await _configure_login_role( + connection, + username=username, + password=password, + default_read_only=True, + ) + await connection.execute( + f"REVOKE ALL ON DATABASE {_quoted_identifier(database)} FROM PUBLIC" + ) + await connection.execute("REVOKE ALL ON SCHEMA public FROM PUBLIC") + await connection.execute("REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC") + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {_quoted_identifier(owner)} " + "IN SCHEMA public REVOKE ALL ON TABLES FROM PUBLIC" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {_quoted_identifier(owner)} " + "IN SCHEMA public REVOKE ALL ON SEQUENCES FROM PUBLIC" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {_quoted_identifier(owner)} " + "IN SCHEMA public REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC" + ) + await _reset_runtime_privileges( + connection, + username=username, + database_name=database, + owner_username=owner, + ) + role = _quoted_identifier(username) + tables = ", ".join(_quoted_identifier(table) for table in READER_TABLES) + await connection.execute(f"GRANT SELECT ON TABLE {tables} TO {role}") + + +async def configure_writer_role( + connection: asyncpg.Connection, + environment: Mapping[str, str], +) -> None: + username = identifier( + _required(environment, "DATABASE_WRITER_USER"), "DATABASE_WRITER_USER" + ) + password = _required(environment, "DATABASE_WRITER_PASSWORD") + owner = identifier( + _required(environment, "DATABASE_ADMIN_USER"), "DATABASE_ADMIN_USER" + ) + database = identifier(_required(environment, "DATABASE_NAME"), "DATABASE_NAME") + await _configure_login_role( + connection, + username=username, + password=password, + default_read_only=False, + ) + await _reset_runtime_privileges( + connection, + username=username, + database_name=database, + owner_username=owner, + ) + role = _quoted_identifier(username) + read_tables = ", ".join( + _quoted_identifier(table) for table in WRITER_READ_TABLES + ) + await connection.execute(f"GRANT SELECT ON TABLE {read_tables} TO {role}") + for table, privileges in WRITER_TABLE_PRIVILEGES.items(): + privilege_list = ", ".join(privileges) + await connection.execute( + f"GRANT {privilege_list} ON TABLE {_quoted_identifier(table)} TO {role}" + ) + for table, grants in WRITER_COLUMN_PRIVILEGES.items(): + for privilege, columns in grants.items(): + column_list = ", ".join(_quoted_identifier(column) for column in columns) + await connection.execute( + f"GRANT {privilege} ({column_list}) ON TABLE " + f"{_quoted_identifier(table)} TO {role}" + ) + + +async def configure_pipeline_role( + connection: asyncpg.Connection, + environment: Mapping[str, str], +) -> None: + username = identifier( + _required(environment, "DATABASE_PIPELINE_USER"), "DATABASE_PIPELINE_USER" + ) + password = _required(environment, "DATABASE_PIPELINE_PASSWORD") + owner = identifier( + _required(environment, "DATABASE_ADMIN_USER"), "DATABASE_ADMIN_USER" + ) + database = identifier(_required(environment, "DATABASE_NAME"), "DATABASE_NAME") + await _configure_login_role( + connection, + username=username, + password=password, + default_read_only=False, + ) + await _reset_runtime_privileges( + connection, + username=username, + database_name=database, + owner_username=owner, + ) + role = _quoted_identifier(username) + tables = ", ".join(_quoted_identifier(table) for table in PIPELINE_READ_TABLES) + await connection.execute(f"GRANT SELECT ON TABLE {tables} TO {role}") + for table, privileges in PIPELINE_TABLE_PRIVILEGES.items(): + await connection.execute( + f"GRANT {', '.join(privileges)} ON TABLE " + f"{_quoted_identifier(table)} TO {role}" + ) + for function in PIPELINE_FUNCTIONS: + await connection.execute(f"GRANT EXECUTE ON FUNCTION {function} TO {role}") + + +def _business_row_checksum(row: Mapping[str, Any], *, identifier_field: str) -> str: + business_fields = dict(row) + business_fields.pop(identifier_field) + canonical = json.dumps( + business_fields, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +async def default_configuration_snapshot( + connection: asyncpg.Connection, +) -> dict[str, dict[str, str]]: + """Return only stable IDs and hashes of explicitly non-sensitive fields.""" + source_rows = await connection.fetch( + f"SELECT {', '.join(SOURCE_CONFIGURATION_INTEGRITY_FIELDS)} " + "FROM source_configurations ORDER BY source_name" + ) + policy_rows = await connection.fetch( + f"SELECT {', '.join(RETENTION_POLICY_INTEGRITY_FIELDS)} " + "FROM retention_policies ORDER BY retention_class" + ) + return { + "source_configurations": { + str(row["source_name"]): _business_row_checksum( + row, identifier_field="source_name" + ) + for row in source_rows + }, + "retention_policies": { + str(row["retention_class"]): _business_row_checksum( + row, identifier_field="retention_class" + ) + for row in policy_rows + }, + } + + +def default_integrity_evidence( + *, + before: Mapping[str, Mapping[str, str]], + after: Mapping[str, Mapping[str, str]], + expected_default_ids: Mapping[str, Iterable[str]], +) -> dict[str, Any]: + evidence: dict[str, Any] = {} + for relation in ("source_configurations", "retention_policies"): + before_rows = dict(before[relation]) + after_rows = dict(after[relation]) + before_ids = set(before_rows) + after_ids = set(after_rows) + configured_ids = set(expected_default_ids[relation]) + expected_added = configured_ids.difference(before_ids) + actual_added = after_ids.difference(before_ids) + existing_preserved = before_ids.issubset(after_ids) + checksums_unchanged = all( + after_rows.get(stable_id) == checksum + for stable_id, checksum in before_rows.items() + ) + only_missing_defaults_added = actual_added == expected_added + relation_evidence = { + "pre_ids": sorted(before_ids), + "post_ids": sorted(after_ids), + "added_ids": sorted(actual_added), + "pre_checksums": { + stable_id: before_rows[stable_id] for stable_id in sorted(before_ids) + }, + "post_checksums_for_pre_ids": { + stable_id: after_rows[stable_id] + for stable_id in sorted(before_ids.intersection(after_ids)) + }, + "existing_ids_preserved": existing_preserved, + "existing_checksums_unchanged": checksums_unchanged, + "only_missing_defaults_added": only_missing_defaults_added, + } + if not all( + ( + existing_preserved, + checksums_unchanged, + only_missing_defaults_added, + ) + ): + raise RuntimeError(f"{relation} bootstrap integrity verification failed") + evidence[relation] = relation_evidence + return evidence + + +async def bootstrap_runtime_configuration( + connection: asyncpg.Connection, + *, + sources: Iterable[SourceConfiguration] | None = None, + governance: GovernanceConfiguration | None = None, +) -> dict[str, int]: + """Insert only absent static defaults; never overwrite legitimate state.""" + governance_configuration = governance or load_governance_configuration() + governance_configuration.validate() + policy_records = [ + policy.database_record(policy_status=governance_configuration.policy_status) + for policy in governance_configuration.policies + ] + inserted_policies = 0 + for record in policy_records: + result = await connection.execute( + """ + INSERT INTO retention_policies ( + retention_class, classification, policy_status, + archive_after_days, delete_after_days, export_expire_after_days, + destructive_deletion_enabled, production_approved, + configuration_checksum + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT (retention_class) DO NOTHING + """, + record["retention_class"], + record["classification"], + record["policy_status"], + record["archive_after_days"], + record["delete_after_days"], + record["export_expire_after_days"], + record["destructive_deletion_enabled"], + record["production_approved"], + record["configuration_checksum"], + ) + inserted_policies += int(result.endswith(" 1")) + + source_records = [ + configuration.database_record() + for configuration in (sources or load_source_configurations()) + ] + inserted_sources = 0 + for record in source_records: + result = await connection.execute( + """ + INSERT INTO source_configurations ( + source_name, source_owner, technical_owner, legal_status, + licence_status, terms_url, rate_limit_per_minute, user_agent, + freshness_sla_hours, schedule_expression, enabled, + governance_blocked, last_success_at, watermark, classification, + retention_class, schema_version, credentials_reference, + retry_limit, timeout_seconds, maximum_pages, maximum_records, + configuration_checksum + ) VALUES ( + $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12, + CAST($13 AS timestamptz),$14,$15,$16,$17,$18,$19,$20,$21,$22,$23 + ) + ON CONFLICT (source_name) DO NOTHING + """, + record["source_name"], + record["source_owner"], + record["technical_owner"], + record["legal_status"], + record["licence_status"], + record["terms_url"], + record["rate_limit_per_minute"], + record["user_agent"], + record["freshness_sla_hours"], + record["schedule_expression"], + record["enabled"], + record["governance_blocked"], + record["last_success"], + record["watermark"], + record["classification"], + record["retention_class"], + record["schema_version"], + record["credentials_reference"], + record["retry_limit"], + record["timeout_seconds"], + record["maximum_pages"], + record["maximum_records"], + record["configuration_checksum"], + ) + inserted_sources += int(result.endswith(" 1")) + return { + "source_configurations_inserted": inserted_sources, + "retention_policies_inserted": inserted_policies, + } + + +async def _connect_runtime( + environment: Mapping[str, str], *, prefix: str +) -> asyncpg.Connection: + return await asyncpg.connect( + host=_required(environment, "DATABASE_HOST"), + port=int(environment.get("DATABASE_PORT", "5432")), + user=_required(environment, f"DATABASE_{prefix}_USER"), + password=_required(environment, f"DATABASE_{prefix}_PASSWORD"), + database=_required(environment, "DATABASE_NAME"), + ssl=environment.get("DATABASE_SSL_MODE", "disable").strip().lower(), + command_timeout=None, + ) + + +async def _statement_denied(connection: asyncpg.Connection, statement: str) -> bool: + transaction = connection.transaction() + await transaction.start() + try: + await connection.execute(statement) + except ( + asyncpg.InsufficientPrivilegeError, + asyncpg.ReadOnlySQLTransactionError, + ): + return True + finally: + await transaction.rollback() + return False + + +async def verify_reader_role(environment: Mapping[str, str]) -> dict[str, bool]: + connection = await _connect_runtime(environment, prefix="READER") + reader = identifier( + _required(environment, "DATABASE_READER_USER"), "DATABASE_READER_USER" + ) + writer = identifier( + _required(environment, "DATABASE_WRITER_USER"), "DATABASE_WRITER_USER" + ) + try: + return { + "analytics_select_succeeded": ( + await connection.fetchval( + "SELECT COUNT(*) >= 0 FROM analytics_scope_totals" + ) + is True + ), + "tls_in_use": bool( + await connection.fetchval( + "SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()" + ) + ), + "default_read_only": ( + await connection.fetchval("SHOW default_transaction_read_only") == "on" + ), + "insert_denied": await _statement_denied( + connection, + "INSERT INTO idempotency_records " + "(actor_id, action, idempotency_key, request_hash, status, expires_at) " + "VALUES ('permission-check','permission-check','permission-check'," + "'permission-check','reserved',CURRENT_TIMESTAMP)", + ), + "update_denied": await _statement_denied( + connection, "UPDATE dataset_versions SET status=status WHERE FALSE" + ), + "delete_denied": await _statement_denied( + connection, "DELETE FROM job_runs WHERE FALSE" + ), + "ddl_denied": await _statement_denied( + connection, "CREATE TABLE fip_reader_permission_probe (id integer)" + ), + "drop_denied": await _statement_denied( + connection, "DROP TABLE charities" + ), + "grant_denied": await _statement_denied( + connection, + f"GRANT {_quoted_identifier(writer)} TO {_quoted_identifier(reader)}", + ), + "create_role_denied": await _statement_denied( + connection, "CREATE ROLE fip_reader_permission_probe" + ), + } + finally: + await connection.close() + + +async def verify_writer_role(environment: Mapping[str, str]) -> dict[str, bool]: + connection = await _connect_runtime(environment, prefix="WRITER") + writer = identifier( + _required(environment, "DATABASE_WRITER_USER"), "DATABASE_WRITER_USER" + ) + reader = identifier( + _required(environment, "DATABASE_READER_USER"), "DATABASE_READER_USER" + ) + probe = f"permission-check-{uuid.uuid4()}" + request_hash = uuid.uuid4().hex + uuid.uuid4().hex + transaction = connection.transaction() + allowed_dml = False + try: + await transaction.start() + await connection.execute( + """ + INSERT INTO idempotency_records ( + actor_id, action, idempotency_key, request_hash, status, expires_at + ) VALUES ($1,'permission-check',$2,$3,'reserved',CURRENT_TIMESTAMP + INTERVAL '1 minute') + """, + probe, + probe, + request_hash, + ) + await connection.execute( + "UPDATE idempotency_records SET status=status WHERE actor_id=$1", + probe, + ) + await connection.execute( + "DELETE FROM idempotency_records WHERE actor_id=$1", probe + ) + await connection.execute( + "UPDATE source_configurations SET enabled=enabled WHERE FALSE" + ) + allowed_dml = True + finally: + await transaction.rollback() + try: + return { + "select_succeeded": await connection.fetchval("SELECT 1") == 1, + "allowed_dml_succeeded": allowed_dml, + "unlisted_dml_denied": await _statement_denied( + connection, "UPDATE charities SET name=name WHERE FALSE" + ), + "unlisted_column_update_denied": await _statement_denied( + connection, + "UPDATE source_configurations " + "SET credentials_reference=credentials_reference WHERE FALSE", + ), + "ddl_denied": await _statement_denied( + connection, "CREATE TABLE fip_writer_permission_probe (id integer)" + ), + "drop_denied": await _statement_denied( + connection, "DROP TABLE charities" + ), + "create_role_denied": await _statement_denied( + connection, "CREATE ROLE fip_writer_permission_probe" + ), + "grant_denied": await _statement_denied( + connection, + f"GRANT {_quoted_identifier(reader)} TO {_quoted_identifier(writer)}", + ), + } + finally: + await connection.close() + + +async def verify_pipeline_role(environment: Mapping[str, str]) -> dict[str, bool]: + connection = await _connect_runtime(environment, prefix="PIPELINE") + pipeline = identifier( + _required(environment, "DATABASE_PIPELINE_USER"), "DATABASE_PIPELINE_USER" + ) + reader = identifier( + _required(environment, "DATABASE_READER_USER"), "DATABASE_READER_USER" + ) + try: + return { + "schema_select_succeeded": ( + await connection.fetchval("SELECT COUNT(*) = 1 FROM alembic_version") + ), + "dataset_update_allowed": not await _statement_denied( + connection, "UPDATE dataset_versions SET status=status WHERE FALSE" + ), + "materialization_execute_allowed": bool( + await connection.fetchval( + "SELECT has_function_privilege($1, " + "'refresh_analytics_materializations(text)', 'EXECUTE')", + pipeline, + ) + ), + "runtime_job_update_denied": await _statement_denied( + connection, "UPDATE job_runs SET status=status WHERE FALSE" + ), + "ddl_denied": await _statement_denied( + connection, "CREATE TABLE fip_pipeline_permission_probe (id integer)" + ), + "drop_denied": await _statement_denied(connection, "DROP TABLE charities"), + "create_role_denied": await _statement_denied( + connection, "CREATE ROLE fip_pipeline_permission_probe" + ), + "grant_denied": await _statement_denied( + connection, + f"GRANT {_quoted_identifier(reader)} TO {_quoted_identifier(pipeline)}", + ), + } + finally: + await connection.close() + + +async def grant_snapshot( + connection: asyncpg.Connection, usernames: Iterable[str] +) -> dict[str, Any]: + principals = tuple(usernames) + table_rows = await connection.fetch( + """ + SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + namespace.nspname AS table_schema, + relation.relname AS table_name, + acl.privilege_type + FROM pg_class AS relation + JOIN pg_namespace AS namespace ON namespace.oid=relation.relnamespace + CROSS JOIN LATERAL aclexplode( + COALESCE(relation.relacl, acldefault('r', relation.relowner)) + ) AS acl + LEFT JOIN pg_roles AS grantee ON grantee.oid=acl.grantee + WHERE namespace.nspname='public' + AND relation.relkind IN ('r', 'p', 'v', 'm', 'f') + AND (acl.grantee=0 OR grantee.rolname=ANY($1::text[])) + ORDER BY grantee, table_name, privilege_type + """, + principals, + ) + column_rows = await connection.fetch( + """ + SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + namespace.nspname AS table_schema, + relation.relname AS table_name, + attribute.attname AS column_name, + acl.privilege_type + FROM pg_attribute AS attribute + JOIN pg_class AS relation ON relation.oid=attribute.attrelid + JOIN pg_namespace AS namespace ON namespace.oid=relation.relnamespace + CROSS JOIN LATERAL aclexplode(attribute.attacl) AS acl + LEFT JOIN pg_roles AS grantee ON grantee.oid=acl.grantee + WHERE namespace.nspname='public' + AND relation.relkind IN ('r', 'p', 'v', 'm', 'f') + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + AND (acl.grantee=0 OR grantee.rolname=ANY($1::text[])) + ORDER BY grantee, table_name, column_name, privilege_type + """, + principals, + ) + sequence_rows = await connection.fetch( + """ + SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + namespace.nspname AS object_schema, + relation.relname AS object_name, + acl.privilege_type + FROM pg_class AS relation + JOIN pg_namespace AS namespace ON namespace.oid=relation.relnamespace + CROSS JOIN LATERAL aclexplode( + COALESCE(relation.relacl, acldefault('s', relation.relowner)) + ) AS acl + LEFT JOIN pg_roles AS grantee ON grantee.oid=acl.grantee + WHERE namespace.nspname='public' + AND relation.relkind='S' + AND (acl.grantee=0 OR grantee.rolname=ANY($1::text[])) + ORDER BY grantee, object_name, privilege_type + """, + principals, + ) + function_rows = await connection.fetch( + """ + SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + namespace.nspname AS object_schema, + routine.proname AS object_name, + extension.extname AS extension_name, + acl.privilege_type + FROM pg_proc AS routine + JOIN pg_namespace AS namespace ON namespace.oid=routine.pronamespace + CROSS JOIN LATERAL aclexplode( + COALESCE(routine.proacl, acldefault('f', routine.proowner)) + ) AS acl + LEFT JOIN pg_roles AS grantee ON grantee.oid=acl.grantee + LEFT JOIN pg_depend AS dependency + ON dependency.classid='pg_proc'::regclass + AND dependency.objid=routine.oid + AND dependency.deptype='e' + LEFT JOIN pg_extension AS extension ON extension.oid=dependency.refobjid + WHERE namespace.nspname='public' + AND (acl.grantee=0 OR grantee.rolname=ANY($1::text[])) + ORDER BY grantee, object_name, privilege_type + """, + principals, + ) + database_rows = await connection.fetch( + """ + SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + database.datname AS database_name, + acl.privilege_type + FROM pg_database AS database + CROSS JOIN LATERAL aclexplode( + COALESCE(database.datacl, acldefault('d', database.datdba)) + ) AS acl + LEFT JOIN pg_roles AS grantee ON grantee.oid=acl.grantee + WHERE database.datname=current_database() + AND (acl.grantee=0 OR grantee.rolname=ANY($1::text[])) + ORDER BY grantee, privilege_type + """, + principals, + ) + schema_rows = await connection.fetch( + """ + SELECT COALESCE(grantee.rolname, 'PUBLIC') AS grantee, + namespace.nspname AS schema_name, + acl.privilege_type + FROM pg_namespace AS namespace + CROSS JOIN LATERAL aclexplode( + COALESCE(namespace.nspacl, acldefault('n', namespace.nspowner)) + ) AS acl + LEFT JOIN pg_roles AS grantee ON grantee.oid=acl.grantee + WHERE namespace.nspname='public' + AND (acl.grantee=0 OR grantee.rolname=ANY($1::text[])) + ORDER BY grantee, privilege_type + """, + principals, + ) + return { + "database_privileges": [dict(row) for row in database_rows], + "schema_privileges": [dict(row) for row in schema_rows], + "table_privileges": [dict(row) for row in table_rows], + "column_privileges": [dict(row) for row in column_rows], + "sequence_privileges": [dict(row) for row in sequence_rows], + "function_privileges": [dict(row) for row in function_rows], + } + + +def _expected_grants( + *, reader: str, writer: str, database_name: str, pipeline: str | None = None +) -> dict[str, set[tuple[str, ...]]]: + table_privileges: set[tuple[str, ...]] = { + (reader, table, "SELECT") for table in READER_TABLES + } + table_privileges.update( + (writer, table, "SELECT") for table in WRITER_READ_TABLES + ) + table_privileges.update( + (writer, table, privilege) + for table, privileges in WRITER_TABLE_PRIVILEGES.items() + for privilege in privileges + ) + column_privileges: set[tuple[str, ...]] = { + (writer, table, column, privilege) + for table, grants in WRITER_COLUMN_PRIVILEGES.items() + for privilege, columns in grants.items() + for column in columns + } + if pipeline: + table_privileges.update( + (pipeline, table, "SELECT") for table in PIPELINE_READ_TABLES + ) + table_privileges.update( + (pipeline, table, privilege) + for table, privileges in PIPELINE_TABLE_PRIVILEGES.items() + for privilege in privileges + ) + database_privileges: set[tuple[str, ...]] = { + (reader, database_name, "CONNECT"), + (writer, database_name, "CONNECT"), + } + schema_privileges: set[tuple[str, ...]] = { + (reader, "public", "USAGE"), + (writer, "public", "USAGE"), + } + if pipeline: + database_privileges.add((pipeline, database_name, "CONNECT")) + schema_privileges.add((pipeline, "public", "USAGE")) + return { + "database_privileges": database_privileges, + "schema_privileges": schema_privileges, + "table_privileges": table_privileges, + "column_privileges": column_privileges, + "sequence_privileges": set(), + "function_privileges": set(), + } + + +def grant_equality_evidence( + snapshot: Mapping[str, Any], *, reader: str, writer: str, + database_name: str, pipeline: str | None = None +) -> dict[str, bool]: + expected = _expected_grants( + reader=reader, writer=writer, database_name=database_name, pipeline=pipeline + ) + projections = { + "database_privileges": ("grantee", "database_name", "privilege_type"), + "schema_privileges": ("grantee", "schema_name", "privilege_type"), + "table_privileges": ("grantee", "table_name", "privilege_type"), + "column_privileges": ( + "grantee", + "table_name", + "column_name", + "privilege_type", + ), + "sequence_privileges": ("grantee", "object_name", "privilege_type"), + "function_privileges": ("grantee", "object_name", "privilege_type"), + } + evidence = { + name: { + tuple(str(row[column]) for column in projections[name]) + for row in snapshot[name] + } + == expected[name] + for name in projections + if name != "function_privileges" + } + evidence["function_privileges"] = all( + ( + str(row["grantee"]) == "PUBLIC" + and str(row["privilege_type"]) == "EXECUTE" + and str(row["extension_name"]) in PUBLIC_FUNCTION_EXTENSION_ALLOWLIST + ) + or ( + pipeline is not None + and str(row["grantee"]) == pipeline + and str(row["privilege_type"]) == "EXECUTE" + and str(row["object_name"]) == "refresh_analytics_materializations" + ) + for row in snapshot["function_privileges"] + ) + evidence["all_grants_equal_allowlist"] = all(evidence.values()) + return evidence + + +async def role_security_evidence( + connection: asyncpg.Connection, + *, + username: str, + database_name: str, +) -> dict[str, Any]: + state = await _required_runtime_role_state(connection, username) + _assert_runtime_role_safe( + state, username=username, require_login_posture=True + ) + owned_rows = await connection.fetch( + """ + SELECT 'schemas' AS object_type, namespace.nspname AS schema_name, + namespace.nspname AS object_name + FROM pg_namespace AS namespace + JOIN pg_roles AS owner ON owner.oid=namespace.nspowner + WHERE owner.rolname=$1 + UNION ALL + SELECT CASE WHEN relation.relkind='S' THEN 'sequences' ELSE 'tables' END, + namespace.nspname, relation.relname + FROM pg_class AS relation + JOIN pg_namespace AS namespace ON namespace.oid=relation.relnamespace + JOIN pg_roles AS owner ON owner.oid=relation.relowner + WHERE owner.rolname=$1 + AND relation.relkind IN ('r', 'p', 'v', 'm', 'f', 'S') + UNION ALL + SELECT 'functions', namespace.nspname, routine.proname + FROM pg_proc AS routine + JOIN pg_namespace AS namespace ON namespace.oid=routine.pronamespace + JOIN pg_roles AS owner ON owner.oid=routine.proowner + WHERE owner.rolname=$1 + ORDER BY object_type, schema_name, object_name + """, + username, + ) + owned_counts = { + object_type: sum( + 1 for row in owned_rows if str(row["object_type"]) == object_type + ) + for object_type in ("schemas", "tables", "sequences", "functions") + } + evidence = { + "rolsuper": bool(state["rolsuper"]), + "rolcreaterole": bool(state["rolcreaterole"]), + "rolcreatedb": bool(state["rolcreatedb"]), + "rolreplication": bool(state["rolreplication"]), + "rolbypassrls": bool(state["rolbypassrls"]), + "rolinherit": bool(state["rolinherit"]), + "rolcanlogin": bool(state["rolcanlogin"]), + "memberships": list(state["memberships"]), + "database_connect": bool( + await connection.fetchval( + "SELECT has_database_privilege($1, $2, 'CONNECT')", + username, + database_name, + ) + ), + "database_create": bool( + await connection.fetchval( + "SELECT has_database_privilege($1, $2, 'CREATE')", + username, + database_name, + ) + ), + "database_temporary": bool( + await connection.fetchval( + "SELECT has_database_privilege($1, $2, 'TEMPORARY')", + username, + database_name, + ) + ), + "schema_usage": bool( + await connection.fetchval( + "SELECT has_schema_privilege($1, 'public', 'USAGE')", username + ) + ), + "schema_create": bool( + await connection.fetchval( + "SELECT has_schema_privilege($1, 'public', 'CREATE')", username + ) + ), + "owned_objects": owned_counts, + } + if ( + evidence["memberships"] + or not evidence["database_connect"] + or evidence["database_create"] + or evidence["database_temporary"] + or not evidence["schema_usage"] + or evidence["schema_create"] + or any(owned_counts.values()) + ): + raise RuntimeError(f"runtime role {username} failed least-privilege evidence") + return evidence + + +async def run(environment: Mapping[str, str] | None = None) -> dict[str, Any]: + task_environment = dict(os.environ if environment is None else environment) + reader = identifier( + _required(task_environment, "DATABASE_READER_USER"), "DATABASE_READER_USER" + ) + writer = identifier( + _required(task_environment, "DATABASE_WRITER_USER"), "DATABASE_WRITER_USER" + ) + pipeline = identifier( + _required(task_environment, "DATABASE_PIPELINE_USER"), "DATABASE_PIPELINE_USER" + ) + if len({reader, writer, pipeline}) != 3: + raise DatabaseAccessConfigurationError( + "Reader, writer and pipeline principals must be distinct" + ) + admin = await connect_admin(task_environment) + try: + acquired = await admin.fetchval( + "SELECT pg_try_advisory_lock($1)", DATABASE_ACCESS_LOCK_ID + ) + if not acquired: + raise DatabaseAccessConfigurationError( + "another database access prerequisite task holds the lock" + ) + for runtime_role in (reader, writer, pipeline): + state = await runtime_role_state(admin, runtime_role) + if state is not None: + _assert_runtime_role_safe(state, username=runtime_role) + + sources = tuple(load_source_configurations()) + governance = load_governance_configuration() + defaults_before = await default_configuration_snapshot(admin) + bootstrap = await bootstrap_runtime_configuration( + admin, sources=sources, governance=governance + ) + defaults_after = await default_configuration_snapshot(admin) + defaults_evidence = default_integrity_evidence( + before=defaults_before, + after=defaults_after, + expected_default_ids={ + "source_configurations": ( + configuration.source_name for configuration in sources + ), + "retention_policies": ( + policy.retention_class for policy in governance.policies + ), + }, + ) + await configure_reader_role(admin, task_environment) + await configure_writer_role(admin, task_environment) + await configure_pipeline_role(admin, task_environment) + snapshot = await grant_snapshot(admin, (reader, writer, pipeline)) + grants_equal = grant_equality_evidence( + snapshot, + reader=reader, + writer=writer, + pipeline=pipeline, + database_name=_required(task_environment, "DATABASE_NAME"), + ) + if not grants_equal["all_grants_equal_allowlist"]: + raise RuntimeError("runtime database grants differ from the allowlist") + role_evidence = { + "reader": await role_security_evidence( + admin, + username=reader, + database_name=_required(task_environment, "DATABASE_NAME"), + ), + "writer": await role_security_evidence( + admin, + username=writer, + database_name=_required(task_environment, "DATABASE_NAME"), + ), + "pipeline": await role_security_evidence( + admin, + username=pipeline, + database_name=_required(task_environment, "DATABASE_NAME"), + ), + } + reader_verification = await verify_reader_role(task_environment) + writer_verification = await verify_writer_role(task_environment) + pipeline_verification = await verify_pipeline_role(task_environment) + if ( + not all(reader_verification.values()) + or not all(writer_verification.values()) + or not all(pipeline_verification.values()) + ): + raise RuntimeError("database access prerequisite verification failed") + return { + "reader": reader, + "writer": writer, + "pipeline": pipeline, + "bootstrap": bootstrap, + "default_integrity": defaults_evidence, + "reader_verification": reader_verification, + "writer_verification": writer_verification, + "pipeline_verification": pipeline_verification, + "role_evidence": role_evidence, + "grant_equality": grants_equal, + "grants": snapshot, + } + finally: + await admin.close() + + +def main() -> int: + result = asyncio.run(run()) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/migration/sqlite_to_postgres.py b/src/migration/sqlite_to_postgres.py index 2bafaa9..2472f6c 100644 --- a/src/migration/sqlite_to_postgres.py +++ b/src/migration/sqlite_to_postgres.py @@ -483,8 +483,10 @@ def _converted_record( return ((dataset_version,) + values) if table in VERSIONED_TABLES else values -async def _connect_postgres() -> asyncpg.Connection: - settings = DatabaseSettings.from_env() +async def _connect_postgres( + settings: DatabaseSettings | None = None, +) -> asyncpg.Connection: + settings = settings or DatabaseSettings.from_env() url = settings.sqlalchemy_url() connection = await asyncpg.connect( host=url.host, @@ -956,7 +958,7 @@ async def _activate( reconciliation: dict[str, dict[str, Any]], staged_global_records: dict[str, list[tuple[Any, ...]]], staged_global_columns: dict[str, tuple[str, ...]], -) -> Optional[str]: +) -> tuple[Optional[str], int]: failures = [name for name, result in reconciliation.items() if result["status"] == "fail"] if failures: await connection.execute( @@ -995,6 +997,21 @@ async def _activate( previous = await connection.fetchval( "SELECT dataset_version FROM dataset_versions WHERE is_active FOR UPDATE" ) + retargeted = await connection.execute( + """ + UPDATE source_funder_link_overrides AS override + SET target_dataset_version=$1, revision=override.revision + 1, + updated_at=CURRENT_TIMESTAMP + WHERE override.link_mode='link_profile' + AND override.target_dataset_version<>$1 + AND EXISTS ( + SELECT 1 FROM charities + WHERE dataset_version=$1 + AND charity_id=override.target_profile_id + ) + """, + dataset_version, + ) if previous: await connection.execute( """ @@ -1024,7 +1041,7 @@ async def _activate( json.dumps(counts, sort_keys=True), json.dumps(reconciliation, sort_keys=True), ) - return previous + return previous, int(retargeted.rsplit(" ", 1)[-1]) async def migrate( @@ -1041,6 +1058,7 @@ async def migrate( enforce_baseline: bool = True, enforce_capacity: bool = True, remote_postgres: bool = False, + database_settings: DatabaseSettings | None = None, ) -> dict[str, Any]: if not re.fullmatch(r"[a-f0-9]{40}", code_revision): raise PreflightError("Code revision must be a full lowercase Git SHA") @@ -1057,7 +1075,7 @@ async def migrate( uuid.NAMESPACE_URL, f"foundation-intelligence:{preflight.checksum}:{dataset_version}:{code_revision}", ) - target = await _connect_postgres() + target = await _connect_postgres(database_settings) started_at = datetime.now(timezone.utc) try: await _ensure_schema(target) @@ -1101,7 +1119,7 @@ async def migrate( EXPECTED_COUNTS if enforce_baseline else None, EXPECTED_CONTROLS if enforce_baseline else None, ) - previous = await _activate( + previous, retargeted_overrides = await _activate( target, dataset_version, migration_run_id, @@ -1129,6 +1147,7 @@ async def migrate( "reconciliation_results": reconciliation, "activation_status": "active", "rollback_dataset_version": previous, + "retargeted_overrides": retargeted_overrides, "errors": [], "actor": {"id": actor_id, "type": actor_type}, "capacity": preflight.capacity.as_dict(), diff --git a/src/pipelines/durable_worker.py b/src/pipelines/durable_worker.py index dbe4ae8..77a773f 100644 --- a/src/pipelines/durable_worker.py +++ b/src/pipelines/durable_worker.py @@ -7,7 +7,7 @@ from typing import Any, Awaitable, Callable, Mapping, Protocol from bff.postgres.job_repository import PostgresJobRepository -from bff.utils.logging import logger +from bff.utils.logging import logger, redact_text JobHandler = Callable[[Mapping[str, Any]], Awaitable[Mapping[str, Any]]] @@ -81,6 +81,30 @@ async def run_once(self) -> WorkerResult: }, ) return WorkerResult(status="failed", job_id=job_id) + heartbeat_stop = asyncio.Event() + + async def maintain_lease() -> None: + interval = max(2.0, self.lease_seconds / 3) + while True: + try: + await asyncio.wait_for(heartbeat_stop.wait(), timeout=interval) + return + except asyncio.TimeoutError: + if not hasattr(self.repository, "heartbeat"): + return + alive = await self.repository.heartbeat( + job_id, + worker_id=self.worker_id, + lease_seconds=self.lease_seconds, + ) + if not alive: + logger.error( + "job_lease_lost", + extra={"job_id": job_id, "operation": str(job["job_type"])}, + ) + return + + heartbeat_task = asyncio.create_task(maintain_lease()) try: async with asyncio.timeout(int(job["timeout_seconds"])): result = await handler(job) @@ -90,7 +114,7 @@ async def run_once(self) -> WorkerResult: worker_id=self.worker_id, error_class="JobTimeout", failure_reason="Worker execution exceeded the durable job timeout", - retryable=True, + retryable=False, ) logger.error( "job_timed_out", @@ -108,8 +132,11 @@ async def run_once(self) -> WorkerResult: job_id, worker_id=self.worker_id, error_class=exc.__class__.__name__, - failure_reason="Worker handler failed; last-good data remains active", - retryable=True, + failure_reason=( + "Worker handler failed; last-good data remains active: " + + redact_text(str(exc))[:1500] + ), + retryable=False, ) logger.error( "job_failed", @@ -122,6 +149,9 @@ async def run_once(self) -> WorkerResult: }, ) return WorkerResult(status=status, job_id=job_id) + finally: + heartbeat_stop.set() + await heartbeat_task succeeded = await self.repository.succeed( job_id, worker_id=self.worker_id, diff --git a/src/pipelines/job_admin.py b/src/pipelines/job_admin.py new file mode 100644 index 0000000..661c3b0 --- /dev/null +++ b/src/pipelines/job_admin.py @@ -0,0 +1,151 @@ +"""Bounded one-off backlog inspection/retirement for worker deployments.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import uuid + +import asyncpg + + +RETIREMENT_REASON = ( + "Job was created before the worker execution service existed and was retired " + "during worker deployment." +) + + +async def _connection() -> asyncpg.Connection: + required = { + name: str(os.getenv(name) or "").strip() + for name in ( + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_NAME", + "DATABASE_ADMIN_USER", + "DATABASE_ADMIN_PASSWORD", + ) + } + missing = [name for name, value in required.items() if not value] + if missing: + raise RuntimeError("Missing database settings: " + ", ".join(missing)) + return await asyncpg.connect( + host=required["DATABASE_HOST"], + port=int(required["DATABASE_PORT"]), + database=required["DATABASE_NAME"], + user=required["DATABASE_ADMIN_USER"], + password=required["DATABASE_ADMIN_PASSWORD"], + ssl=str(os.getenv("DATABASE_SSL_MODE") or "require"), + command_timeout=30, + ) + + +async def inspect_backlog() -> dict[str, object]: + connection = await _connection() + try: + summary = await connection.fetch( + """ + SELECT job_type, status, COUNT(*) AS count + FROM job_runs + GROUP BY job_type, status + ORDER BY job_type, status + """ + ) + active = await connection.fetch( + """ + SELECT job_run_id, job_type, status, requested_at, started_at, + heartbeat_at, lease_expires_at + FROM job_runs + WHERE status IN ('queued', 'running') + ORDER BY requested_at, job_run_id + LIMIT 200 + """ + ) + return { + "summary": [dict(row) for row in summary], + "active": [dict(row) for row in active], + "active_truncated": len(active) == 200, + } + finally: + await connection.close() + + +async def retire_jobs(job_ids: list[uuid.UUID]) -> dict[str, object]: + if not job_ids or len(job_ids) > 200: + raise ValueError("Retirement requires between one and 200 explicit job IDs") + connection = await _connection() + try: + async with connection.transaction(): + rows = await connection.fetch( + """ + UPDATE job_runs + SET status='cancelled', + started_at=COALESCE(started_at, requested_at), + completed_at=CURRENT_TIMESTAMP, + heartbeat_at=NULL, lease_expires_at=NULL, + error_class='PreWorkerDeploymentRetirement', + error_message=$2, failure_reason=$2 + WHERE job_run_id=ANY($1::uuid[]) + AND status IN ('queued', 'running') + RETURNING job_run_id, job_type, status + """, + job_ids, + RETIREMENT_REASON, + ) + retired = {row["job_run_id"] for row in rows} + for row in rows: + await connection.execute( + """ + INSERT INTO job_events ( + job_event_id, job_run_id, sequence, event_type, + actor_id, details + ) VALUES ( + $1, $2, + (SELECT COALESCE(MAX(sequence), 0) + 1 + FROM job_events WHERE job_run_id=$2), + 'cancelled', 'worker-deployment', $3::jsonb + ) + """, + uuid.uuid4(), + row["job_run_id"], + json.dumps({"reason": RETIREMENT_REASON}), + ) + await connection.execute( + """ + UPDATE worker_heartbeats + SET status='stopped', job_run_id=NULL, + heartbeat_at=CURRENT_TIMESTAMP + WHERE job_run_id=ANY($1::uuid[]) + """, + list(retired), + ) + return { + "requested": [str(job_id) for job_id in job_ids], + "retired": [str(row["job_run_id"]) for row in rows], + "retired_count": len(rows), + } + finally: + await connection.close() + + +async def run(arguments: argparse.Namespace) -> dict[str, object]: + if arguments.action == "inspect": + return await inspect_backlog() + return await retire_jobs([uuid.UUID(value) for value in arguments.job_id]) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Inspect or retire explicit durable jobs") + subparsers = parser.add_subparsers(dest="action", required=True) + subparsers.add_parser("inspect") + retire = subparsers.add_parser("retire") + retire.add_argument("--job-id", action="append", required=True) + result = asyncio.run(run(parser.parse_args())) + print(json.dumps(result, sort_keys=True, default=str)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pipelines/run_pipeline.py b/src/pipelines/run_pipeline.py index b50978b..7487b50 100644 --- a/src/pipelines/run_pipeline.py +++ b/src/pipelines/run_pipeline.py @@ -83,6 +83,7 @@ def publish_full_run_database_with_ecb_conversion( staging_db_file: str, db_file: str, timeout: int, + report_path: str | None = None, ) -> dict: """Validate and convert one staging DB, then publish it exactly once. @@ -95,11 +96,12 @@ def publish_full_run_database_with_ecb_conversion( repair = db_loader.prepare_staging_database_for_publish(staging_db_file) logger.info("Step F: Staging profile-link repair completed: %s", repair) logger.info("Step G: Applying official ECB EUR conversions in staging...") - report_path = os.path.join( - PROJECT_ROOT, - "data/processed/ecb_exchange_rate_backfill_report.json", + conversion_report_path = report_path or os.path.join( + PROJECT_ROOT, "data/processed/ecb_exchange_rate_backfill_report.json" + ) + report = apply_ecb_conversion_backfill( + staging_db_file, conversion_report_path, timeout ) - report = apply_ecb_conversion_backfill(staging_db_file, report_path, timeout) # backfill_database itself atomically replaces the staging path with its # converted clone; validate it again and make the only active publish. db_loader.publish_staging_database(staging_db_file, db_file) @@ -120,13 +122,27 @@ def run_pipeline(args): logger.info(f"Starting {source.upper()} Data Pipeline Execution") logger.info("=========================================") - # Setup Paths - raw_cc_path = args.raw_cc_output or os.path.join(PROJECT_ROOT, "data/raw/register_of_charities_results.json") - raw_ts_path = args.raw_ts_output or os.path.join(PROJECT_ROOT, "data/raw/threesixtygiving_results.json") - raw_philea_path = os.path.join(PROJECT_ROOT, "data/raw/philea_members.json") - charities_jsonl_path = os.path.join(PROJECT_ROOT, "data/preprocessed/charities.jsonl") - grants_jsonl_path = os.path.join(PROJECT_ROOT, "data/preprocessed/grants.jsonl") - db_file = os.path.join(PROJECT_ROOT, "data/charities.db") + # Production workers receive an isolated writable directory. Developer CLI + # behavior remains unchanged when no work directory is supplied. + work_directory = getattr(args, "work_directory", None) + data_root = ( + os.path.join(os.path.abspath(work_directory), "data") + if work_directory + else os.path.join(PROJECT_ROOT, "data") + ) + raw_root = os.path.join(data_root, "raw") + preprocessed_root = os.path.join(data_root, "preprocessed") + processed_root = os.path.join(data_root, "processed") + os.makedirs(raw_root, exist_ok=True) + os.makedirs(preprocessed_root, exist_ok=True) + os.makedirs(processed_root, exist_ok=True) + raw_cc_path = args.raw_cc_output or os.path.join(raw_root, "register_of_charities_results.json") + raw_ts_path = args.raw_ts_output or os.path.join(raw_root, "threesixtygiving_results.json") + raw_philea_path = os.path.join(raw_root, "philea_members.json") + charities_jsonl_path = os.path.join(preprocessed_root, "charities.jsonl") + grants_jsonl_path = os.path.join(preprocessed_root, "grants.jsonl") + db_file = os.path.join(data_root, "charities.db") + successful_primary_imports = 0 if source == "register_of_charities": existing = [] if args.fresh else load_existing_raw(raw_cc_path) @@ -367,11 +383,11 @@ def run_pipeline(args): logger.info(f"Exported grants table to: {grants_jsonl_path}") enrichment_report = build_enrichment_report(charities_list, grants_list) - enrichment_report_path = os.path.join(PROJECT_ROOT, "data/preprocessed/enrichment_report.json") + enrichment_report_path = os.path.join(preprocessed_root, "enrichment_report.json") with open(enrichment_report_path, "w", encoding="utf-8") as f: json.dump(enrichment_report, f, ensure_ascii=False, indent=2) logger.info(f"Exported enrichment coverage report to: {enrichment_report_path}") - philea_report_path = os.path.join(PROJECT_ROOT, "data/preprocessed/philea_integration_report.json") + philea_report_path = os.path.join(preprocessed_root, "philea_integration_report.json") with open(philea_report_path, "w", encoding="utf-8") as f: json.dump(philea_report, f, ensure_ascii=False, indent=2) logger.info(f"Exported Philea integration report to: {philea_report_path}") @@ -465,7 +481,11 @@ def run_pipeline(args): "Fresh full run has no successful primary-source discovery; refusing to publish an empty or Philea-only database." ) logger.info("No foundations to process; active database remains unchanged.") - return + return { + "status": "no_changes", + "database_path": db_file, + "successful_primary_imports": 0, + } logger.info(f"Found {len(target_tuples)} target foundations to process.") @@ -478,7 +498,7 @@ def run_pipeline(args): successful_primary_imports = 0 for idx, (reg_no, suffix) in enumerate(target_tuples, 1): - if reg_no in completed_numbers and not args.fresh: + if reg_no in completed_numbers and not args.fresh and not args.reg_numbers: logger.info(f"[{idx}/{len(target_tuples)}] Skipping Charity {reg_no} (already processed in database).") continue @@ -591,6 +611,7 @@ def run_pipeline(args): staging_db_file, db_file, args.timeout, + os.path.join(processed_root, "ecb_exchange_rate_backfill_report.json"), ) else: @@ -600,6 +621,11 @@ def run_pipeline(args): logger.info("=========================================") logger.info(f"Pipeline Source '{source.upper()}' Completed Successfully!") logger.info("=========================================") + return { + "status": "success", + "database_path": db_file, + "successful_primary_imports": successful_primary_imports, + } if __name__ == "__main__": parser = argparse.ArgumentParser(description="Orchestrate UK Charities and 360Giving Data Pipelines.") @@ -610,6 +636,12 @@ def run_pipeline(args): default="consolidate", help="Pipeline phase to execute." ) + parser.add_argument( + "--work-directory", + type=str, + default=None, + help="Isolated writable root for worker execution artifacts.", + ) parser.add_argument( "--raw-cc-output", type=str, diff --git a/src/pipelines/worker.py b/src/pipelines/worker.py new file mode 100644 index 0000000..60927a9 --- /dev/null +++ b/src/pipelines/worker.py @@ -0,0 +1,92 @@ +"""Long-running ECS/local entrypoint for PostgreSQL-backed pipeline jobs.""" + +from __future__ import annotations + +import asyncio +import os +import signal +import socket + +from sqlalchemy import text + +from bff.database import DatabaseAccess +from bff.postgres.job_repository import PostgresJobRepository +from bff.utils.logging import logger +from observability.metrics import load_observability_configuration +from pipelines.durable_worker import DurableWorker +from pipelines.worker_handlers import build_handlers + + +def _bounded_float(name: str, default: float, minimum: float, maximum: float) -> float: + value = float(os.getenv(name, str(default))) + if value < minimum or value > maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}") + return value + + +async def run() -> None: + if os.getenv("DATA_RUNTIME_MODE", "").strip().lower() != "postgresql": + raise RuntimeError("The production worker requires DATA_RUNTIME_MODE=postgresql") + database = DatabaseAccess.from_env() + if not database.writer_configured: + raise RuntimeError("The production worker requires the restricted runtime writer") + if not database.pipeline_configured: + raise RuntimeError("The production worker requires the dedicated dataset publisher") + sessions = database.write_sessions() + pipeline_sessions = database.pipeline_sessions() + expected_schema = load_observability_configuration().expected_schema_version + async with pipeline_sessions() as session: + actual_schema = await session.scalar(text("SELECT version_num FROM alembic_version")) + if actual_schema != expected_schema: + await database.close() + raise RuntimeError( + f"Worker schema gate failed: expected {expected_schema}, received {actual_schema}" + ) + + worker_id = f"{socket.gethostname()}-{os.getpid()}" + queue_name = os.getenv("WORKER_QUEUE_NAME", "pipeline").strip() or "pipeline" + lease_seconds = int(_bounded_float("WORKER_LEASE_SECONDS", 90, 15, 3600)) + idle_seconds = _bounded_float("WORKER_IDLE_SECONDS", 2, 0.25, 60) + repository = PostgresJobRepository(sessions) + handlers = build_handlers(pipeline_sessions) + worker = DurableWorker( + repository, + handlers.mapping, + worker_id=worker_id, + queue_name=queue_name, + lease_seconds=lease_seconds, + ) + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for handled_signal in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(handled_signal, stop.set) + logger.info( + "pipeline_worker_started", + extra={"worker_id": worker_id, "queue": queue_name}, + ) + try: + while not stop.is_set(): + expired = await repository.fail_expired(queue_name=queue_name) + if expired["failed"]: + logger.warning( + "stale_jobs_failed", + extra={"worker_id": worker_id, "count": expired["failed"]}, + ) + result = await worker.run_once() + if result.status == "idle": + try: + await asyncio.wait_for(stop.wait(), timeout=idle_seconds) + except asyncio.TimeoutError: + pass + finally: + logger.info("pipeline_worker_stopped", extra={"worker_id": worker_id}) + await database.close() + + +def main() -> int: + asyncio.run(run()) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/pipelines/worker_handlers.py b/src/pipelines/worker_handlers.py new file mode 100644 index 0000000..073a6e9 --- /dev/null +++ b/src/pipelines/worker_handlers.py @@ -0,0 +1,431 @@ +"""Production handlers for durable PostgreSQL pipeline jobs.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +import os +from pathlib import Path +import shutil +from types import SimpleNamespace +from typing import Any, Mapping, Protocol +from urllib.parse import urlparse +import sqlite3 +import tempfile + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from bff.database import DatabaseSettings +from bff.postgres.funder_repository import SourceFunderRepository +from bff.postgres.registry_repository import RegistryRepository +from migration.sqlite_to_postgres import migrate +from pipelines.run_pipeline import run_pipeline + + +class WorkerConfigurationError(RuntimeError): + """Raised before a job mutates data when worker configuration is incomplete.""" + + +PIPELINE_ARTIFACT_LOCK_ID = 2_083_370_803_872 + + +class PipelineArtifactStore(Protocol): + def download_baseline(self, destination: Path) -> str: ... + + def publish_snapshot(self, source: Path, *, checksum: str, job_id: str) -> None: ... + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _s3_location(uri: str) -> tuple[str, str]: + parsed = urlparse(uri) + if parsed.scheme != "s3" or not parsed.netloc or not parsed.path.lstrip("/"): + raise WorkerConfigurationError("Pipeline artifact URI must be a bounded s3:// URI") + return parsed.netloc, parsed.path.lstrip("/") + + +@dataclass(frozen=True) +class S3PipelineArtifactStore: + current_uri: str + fallback_uri: str + fallback_checksum: str + + def _client(self) -> Any: + import boto3 + + return boto3.client("s3") + + def download_baseline(self, destination: Path) -> str: + client = self._client() + current_bucket, current_key = _s3_location(self.current_uri) + fallback_bucket, fallback_key = _s3_location(self.fallback_uri) + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + expected_checksum: str | None = None + try: + metadata = client.head_object(Bucket=current_bucket, Key=current_key) + expected_checksum = str(metadata.get("Metadata", {}).get("sha256") or "") + client.download_file(current_bucket, current_key, str(destination)) + except Exception as exc: + from botocore.exceptions import ClientError + + if not isinstance(exc, ClientError): + raise + code = str(exc.response.get("Error", {}).get("Code", "")) + if code not in {"404", "NoSuchKey", "NotFound"}: + raise + client.download_file(fallback_bucket, fallback_key, str(destination)) + expected_checksum = self.fallback_checksum + actual_checksum = sha256_file(destination) + if not expected_checksum or actual_checksum != expected_checksum: + destination.unlink(missing_ok=True) + raise WorkerConfigurationError("Pipeline baseline checksum verification failed") + return actual_checksum + + def publish_snapshot(self, source: Path, *, checksum: str, job_id: str) -> None: + bucket, key = _s3_location(self.current_uri) + self._client().upload_file( + str(source), + bucket, + key, + ExtraArgs={ + "ServerSideEncryption": "AES256", + "Metadata": {"sha256": checksum, "job-id": job_id}, + }, + ) + + +@dataclass(frozen=True) +class LocalPipelineArtifactStore: + """Checksum-verified local snapshot store used only by Docker development.""" + + current_path: Path + fallback_path: Path + fallback_checksum: str + + @property + def checksum_path(self) -> Path: + return self.current_path.with_suffix(self.current_path.suffix + ".sha256") + + def download_baseline(self, destination: Path) -> str: + source = self.current_path if self.current_path.is_file() else self.fallback_path + if not source.is_file(): + raise WorkerConfigurationError("Local pipeline baseline is unavailable") + expected = ( + self.checksum_path.read_text(encoding="utf-8").strip() + if source == self.current_path and self.checksum_path.is_file() + else self.fallback_checksum + ) + if len(expected) != 64: + raise WorkerConfigurationError("Local pipeline baseline checksum is unavailable") + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + shutil.copyfile(source, destination) + actual = sha256_file(destination) + if actual != expected: + destination.unlink(missing_ok=True) + raise WorkerConfigurationError("Local pipeline baseline checksum verification failed") + return actual + + def publish_snapshot(self, source: Path, *, checksum: str, job_id: str) -> None: + del job_id + self.current_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = self.current_path.with_suffix(self.current_path.suffix + ".tmp") + checksum_temporary = self.checksum_path.with_suffix( + self.checksum_path.suffix + ".tmp" + ) + shutil.copyfile(source, temporary) + checksum_temporary.write_text(checksum + "\n", encoding="utf-8") + os.replace(temporary, self.current_path) + os.replace(checksum_temporary, self.checksum_path) + + +def _clear_snapshot_runtime_state(path: Path) -> None: + """Keep interactive global state authoritative in RDS, not in snapshots.""" + connection = sqlite3.connect(path) + try: + connection.execute("DELETE FROM source_funder_link_overrides") + connection.execute("DELETE FROM source_funder_profile_cache") + connection.commit() + finally: + connection.close() + + +class WorkerHandlers: + def __init__( + self, + sessions: async_sessionmaker[AsyncSession], + *, + pipeline_settings: DatabaseSettings | None, + artifact_store: PipelineArtifactStore | None, + source_schema_version: str | None, + code_revision: str | None, + ) -> None: + self.sessions = sessions + self.pipeline_settings = pipeline_settings + self.artifact_store = artifact_store + self.source_schema_version = str(source_schema_version or "").strip() + self.code_revision = str(code_revision or "").strip() + + @property + def mapping(self): + return { + "source_funder_profile_hydration": self.source_funder_profile_hydration, + "source_funder_enrichment": self.source_funder_enrichment, + "registry_enrichment": self.registry_enrichment, + "full_run": self.full_run, + } + + @asynccontextmanager + async def _artifact_lock(self): + """Serialize mutable snapshot work across overlapping ECS deployments.""" + async with self.sessions() as session, session.begin(): + await session.execute( + text("SELECT pg_advisory_xact_lock(:lock_id)"), + {"lock_id": PIPELINE_ARTIFACT_LOCK_ID}, + ) + yield + + async def _persist_snapshot_profile(self, profile_ids: list[int], job_id: str) -> None: + if self.artifact_store is None: + raise WorkerConfigurationError("Pipeline snapshot configuration is missing") + with tempfile.TemporaryDirectory(prefix="profile-snapshot-", dir=os.getenv("TMPDIR")) as temporary: + snapshot = Path(temporary) / "charities.db" + await asyncio.to_thread(self.artifact_store.download_baseline, snapshot) + from bff.charity import _fast_link_confirmed_profiles + + profiles = await asyncio.to_thread( + _fast_link_confirmed_profiles, profile_ids, str(snapshot) + ) + if len(profiles) != len(profile_ids): + raise RuntimeError("Snapshot profile mutation was incomplete") + checksum = await asyncio.to_thread(sha256_file, snapshot) + await asyncio.to_thread( + self.artifact_store.publish_snapshot, + snapshot, + checksum=checksum, + job_id=job_id, + ) + + async def _infer_source_targets( + self, reg_numbers: list[int] + ) -> list[tuple[str, int]]: + identifiers = [f"GB-CHC-{number}" for number in reg_numbers] + async with self.sessions() as session: + dataset_version = await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + rows = ( + await session.execute( + text( + """ + SELECT DISTINCT source_funder_key, source_organization_id + FROM grant_source_funder_facts + WHERE dataset_version=:dataset_version + AND source_organization_id=ANY(CAST(:identifiers AS text[])) + ORDER BY source_funder_key + """ + ), + {"dataset_version": dataset_version, "identifiers": identifiers}, + ) + ).mappings().all() + return [ + (str(row["source_funder_key"]), int(str(row["source_organization_id"]).rsplit("-", 1)[-1])) + for row in rows + ] + + async def source_funder_enrichment( + self, job: Mapping[str, Any] + ) -> Mapping[str, Any]: + payload = dict(job.get("input") or {}) + targets = [ + (str(item.get("source_funder_key") or "").strip(), int(item.get("profile_id") or 0)) + for item in payload.get("targets") or [] + ] + targets = [(key, profile_id) for key, profile_id in targets if key and profile_id > 0] + if not targets: + numbers = sorted({int(value) for value in payload.get("reg_numbers") or []}) + targets = await self._infer_source_targets(numbers) + if not targets: + raise ValueError("No active-dataset source funder matches the confirmed profile") + if len(targets) > 5: + raise ValueError("A source-funder enrichment job may contain at most five targets") + repository = SourceFunderRepository(self.sessions) + registry = RegistryRepository(self.sessions) + actor_id = str(job.get("requested_by") or "pipeline-worker") + profiles: list[dict[str, Any]] = [] + async with self._artifact_lock(): + await self._persist_snapshot_profile( + sorted({profile_id for _, profile_id in targets}), str(job["job_id"]) + ) + for source_funder_key, profile_id in targets: + exact_profile = await registry.link_exact_profile( + profile_id, actor_id=actor_id + ) + if exact_profile is None: + raise ValueError( + "No cached Charity Commission record matches the confirmed profile" + ) + linked = await repository.relink( + source_funder_key, profile_id, actor_id=actor_id + ) + if linked is None: + raise ValueError("The source-funder target no longer exists") + async with self.sessions() as verification_session: + effective = await repository._effective_profile( + verification_session, source_funder_key + ) + if effective is None or int(effective["profile_id"]) != profile_id: + raise RuntimeError("Source-funder profile mutation was not persisted") + profiles.append({**dict(linked), **exact_profile}) + return { + "record_count": len(profiles), + "accepted_count": len(profiles), + "profiles": profiles, + } + + async def source_funder_profile_hydration( + self, job: Mapping[str, Any] + ) -> Mapping[str, Any]: + source_funder_key = str( + (job.get("input") or {}).get("source_funder_key") or "" + ).strip() + if not source_funder_key: + raise ValueError("Profile hydration requires a source funder key") + hydrated = await SourceFunderRepository(self.sessions).hydrate_profile_cache( + source_funder_key, + job_id=str(job["job_id"]), + ) + return { + "record_count": 1, + "accepted_count": 1, + "source_funder_key": hydrated["source_funder_key"], + "profile_id": hydrated["profile_id"], + } + + async def registry_enrichment(self, job: Mapping[str, Any]) -> Mapping[str, Any]: + numbers = sorted({int(value) for value in (job.get("input") or {}).get("reg_numbers") or []}) + if len(numbers) != 1: + raise ValueError("Registry enrichment requires exactly one charity number") + async with self._artifact_lock(): + await self._persist_snapshot_profile(numbers, str(job["job_id"])) + linked = await RegistryRepository(self.sessions).link_exact_profile( + numbers[0], actor_id=str(job.get("requested_by") or "pipeline-worker") + ) + if linked is None: + raise ValueError("No exact active-dataset registry/profile pair exists") + return {"record_count": 1, "accepted_count": 1, "profiles": [linked]} + + async def full_run(self, job: Mapping[str, Any]) -> Mapping[str, Any]: + if self.pipeline_settings is None or self.artifact_store is None: + raise WorkerConfigurationError("Full pipeline publisher configuration is missing") + if len(self.code_revision) != 40 or any( + character not in "0123456789abcdef" for character in self.code_revision + ): + raise WorkerConfigurationError("CODE_REVISION must be a full lowercase Git SHA") + if not self.source_schema_version: + raise WorkerConfigurationError("PIPELINE_SOURCE_SCHEMA_VERSION is required") + payload = dict(job.get("input") or {}) + if payload.get("fresh"): + raise WorkerConfigurationError( + "Fresh full runs are disabled for the versioned RDS publisher" + ) + job_id = str(job["job_id"]) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + dataset_version = f"worker-{timestamp}-{job_id}" + async with self._artifact_lock(): + with tempfile.TemporaryDirectory(prefix="pipeline-", dir=os.getenv("TMPDIR")) as temporary: + work_root = Path(temporary) + database_path = work_root / "data" / "charities.db" + await asyncio.to_thread(self.artifact_store.download_baseline, database_path) + arguments = SimpleNamespace( + source="full_run", + work_directory=str(work_root), + raw_cc_output=None, + raw_ts_output=None, + reg_numbers=payload.get("reg_numbers"), + search=payload.get("search_term"), + org_ids=None, + all_orgs=False, + limit=payload.get("limit"), + sleep=1.0, + timeout=10.0, + skip_scrape=False, + skip_contact_crawler=bool( + payload.get("skip_contact_crawler", False) + ), + fresh=False, + ) + pipeline_result = await asyncio.to_thread(run_pipeline, arguments) + imported = int((pipeline_result or {}).get("successful_primary_imports") or 0) + if imported < 1: + raise RuntimeError("Full pipeline completed without a primary-source mutation") + await asyncio.to_thread(_clear_snapshot_runtime_state, database_path) + checksum = await asyncio.to_thread(sha256_file, database_path) + await asyncio.to_thread( + self.artifact_store.publish_snapshot, + database_path, + checksum=checksum, + job_id=job_id, + ) + report = await migrate( + database_path, + checksum, + self.source_schema_version, + dataset_version, + self.code_revision, + str(job.get("requested_by") or "pipeline-worker"), + "service", + work_root / "evidence", + enforce_baseline=False, + remote_postgres=True, + database_settings=self.pipeline_settings, + ) + return { + "dataset_version": dataset_version, + "record_count": int(report["target_counts"].get("grants", 0)), + "accepted_count": imported, + "activation_status": report["activation_status"], + "source_database_checksum": checksum, + "retargeted_overrides": int(report.get("retargeted_overrides") or 0), + } + + +def build_handlers( + sessions: async_sessionmaker[AsyncSession], + environment: Mapping[str, str] | None = None, +) -> WorkerHandlers: + env = os.environ if environment is None else environment + current_uri = str(env.get("PIPELINE_SNAPSHOT_S3_URI") or "").strip() + fallback_uri = str(env.get("PIPELINE_FALLBACK_S3_URI") or "").strip() + fallback_checksum = str(env.get("PIPELINE_FALLBACK_SHA256") or "").strip() + artifact_store: PipelineArtifactStore | None = None + local_current = str(env.get("PIPELINE_SNAPSHOT_PATH") or "").strip() + local_fallback = str(env.get("PIPELINE_BASELINE_PATH") or "").strip() + if local_current and local_fallback and fallback_checksum: + artifact_store = LocalPipelineArtifactStore( + current_path=Path(local_current), + fallback_path=Path(local_fallback), + fallback_checksum=fallback_checksum, + ) + elif current_uri and fallback_uri and fallback_checksum: + artifact_store = S3PipelineArtifactStore( + current_uri=current_uri, + fallback_uri=fallback_uri, + fallback_checksum=fallback_checksum, + ) + return WorkerHandlers( + sessions, + pipeline_settings=DatabaseSettings.pipeline_from_env(env), + artifact_store=artifact_store, + source_schema_version=env.get("PIPELINE_SOURCE_SCHEMA_VERSION"), + code_revision=env.get("CODE_REVISION"), + ) diff --git a/src/tests/test_aws_migration_entrypoint.py b/src/tests/test_aws_migration_entrypoint.py index ce8bc7b..bbd6308 100644 --- a/src/tests/test_aws_migration_entrypoint.py +++ b/src/tests/test_aws_migration_entrypoint.py @@ -20,8 +20,29 @@ async def fetchval(self, statement: str, *_args): self.statements.append(statement) return self.role_exists - async def execute(self, statement: str): + async def fetchrow(self, statement: str, *_args): + self.statements.append(statement) + if not self.role_exists: + return None + return { + "rolname": "foundation_app", + "rolsuper": False, + "rolcreaterole": False, + "rolcreatedb": False, + "rolreplication": False, + "rolbypassrls": False, + "rolinherit": False, + "rolcanlogin": True, + } + + async def fetch(self, statement: str, *_args): + self.statements.append(statement) + return [] + + async def execute(self, statement: str, *_args): self.statements.append(statement) + if statement.startswith("CREATE ROLE"): + self.role_exists = True class _VerifiedApplicationConnection: @@ -116,14 +137,19 @@ async def test_application_role_is_select_only_and_default_read_only(self): }, ) sql = "\n".join(connection.statements) - self.assertIn("NOSUPERUSER NOCREATEDB NOCREATEROLE", sql) - self.assertIn("GRANT SELECT ON ALL TABLES", sql) + self.assertIn("CREATE ROLE", sql) + self.assertIn("NOCREATEDB NOCREATEROLE", sql) + self.assertNotIn("SUPERUSER", sql) + self.assertIn("GRANT SELECT ON TABLE", sql) + self.assertIn('"source_configurations"', sql) self.assertIn("REVOKE ALL ON ALL SEQUENCES", sql) self.assertIn("REVOKE EXECUTE ON ALL FUNCTIONS", sql) self.assertIn("default_transaction_read_only = on", sql) + self.assertNotIn("GRANT SELECT ON ALL TABLES", sql) + self.assertNotIn("GRANT ALL", sql) self.assertNotIn("GRANT SELECT, INSERT", sql) - async def test_existing_role_is_demoted_before_reuse(self): + async def test_existing_safe_role_is_reconfigured_without_superuser_attribute(self): connection = _RecordingConnection(role_exists=True) await _configure_application_role( connection, # type: ignore[arg-type] @@ -139,7 +165,8 @@ async def test_existing_role_is_demoted_before_reuse(self): for statement in connection.statements if statement.startswith("ALTER ROLE \"foundation_app\" LOGIN") ) - self.assertIn("NOSUPERUSER NOCREATEDB NOCREATEROLE", alter_role) + self.assertIn("LOGIN NOINHERIT PASSWORD", alter_role) + self.assertNotIn("SUPERUSER", alter_role) def test_unsafe_identifiers_fail_before_sql(self): with self.assertRaises(AwsMigrationConfigurationError): diff --git a/src/tests/test_cloudformation_demo.py b/src/tests/test_cloudformation_demo.py new file mode 100644 index 0000000..6a4215e --- /dev/null +++ b/src/tests/test_cloudformation_demo.py @@ -0,0 +1,266 @@ +from pathlib import Path +import json +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +TEMPLATE = ROOT / "infra" / "cloudformation" / "demo.yaml" +DB_PREREQUISITE = ROOT / "infra" / "cloudformation" / "db-access-prerequisite.yaml" +PARAMETERS = ROOT / "infra" / "cloudformation" / "parameters.demo.example.json" +NGINX = ROOT / "docker" / "frontend-nginx.ecs.conf" +MANAGED_CACHING_DISABLED = "4135ea2d-6df8-44a3-9df3-4b5a84be39ad" +MANAGED_ALL_VIEWER_EXCEPT_HOST = "b689b0a8-53d0-40ab-baf2-68738e2966ac" + + +def _block(source: str, name: str) -> str: + match = re.search( + rf"^ {re.escape(name)}:\n(?P(?:(?!^ \S).*(?:\n|$))*)", + source, + flags=re.MULTILINE, + ) + if match is None: + raise AssertionError(f"CloudFormation block {name} was not found") + return match.group(0) + + +class TestCloudFormationCustomerAccessContract(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.template = TEMPLATE.read_text(encoding="utf-8") + cls.db_prerequisite = DB_PREREQUISITE.read_text(encoding="utf-8") + cls.parameters = json.loads(PARAMETERS.read_text(encoding="utf-8")) + cls.nginx = NGINX.read_text(encoding="utf-8") + + def test_deployment_states_and_prefix_list_assertion(self): + lockdown = _block(self.template, "OriginLockdownEnabled") + prefix = _block(self.template, "CloudFrontOriginPrefixListId") + rule = _block(self.template, "RequireCloudFrontPrefixListForLockdown") + self.assertIn("Default: 'false'", lockdown) + self.assertIn("AllowedValues: ['false', 'true']", lockdown) + self.assertIn("Default: ''", prefix) + self.assertIn("RuleCondition: !Equals [!Ref OriginLockdownEnabled, 'true']", rule) + self.assertIn("!Ref CloudFrontOriginPrefixListId", rule) + self.assertNotRegex(prefix, r"pl-[a-f0-9]{8,}") + + def test_state_a_and_b_use_mutually_exclusive_ingress(self): + public = _block(self.template, "PublicAlbHttpIngress") + cloudfront = _block(self.template, "CloudFrontAlbHttpIngress") + self.assertIn("Condition: PublicOriginAccess", public) + self.assertIn("CidrIp: 0.0.0.0/0", public) + self.assertIn("FromPort: 80", public) + self.assertIn("Condition: OriginLockdown", cloudfront) + self.assertIn("SourcePrefixListId: !Ref CloudFrontOriginPrefixListId", cloudfront) + self.assertIn("FromPort: 80", cloudfront) + self.assertNotIn("CidrIp: 0.0.0.0/0", cloudfront) + + def test_listener_forwards_in_a_and_defaults_to_403_in_b(self): + listener = _block(self.template, "HttpListener") + verification = _block(self.template, "OriginVerificationListenerRule") + self.assertIn("!If", listener) + self.assertIn("OriginLockdown", listener) + self.assertIn("Type: fixed-response", listener) + self.assertIn("StatusCode: '403'", listener) + self.assertIn("Type: forward", listener) + self.assertIn("HttpHeaderName: X-FIP-Origin-Verify", verification) + self.assertIn("Values: [!Ref OriginVerificationToken]", verification) + self.assertIn("TargetGroupArn: !Ref FrontendTargetGroup", verification) + + def test_cloudfront_default_and_api_behaviors(self): + distribution = _block(self.template, "CloudFrontDistribution") + self.assertIn("CloudFrontDefaultCertificate: true", distribution) + self.assertIn("ViewerProtocolPolicy: redirect-to-https", distribution) + self.assertIn("DefaultRootObject: index.html", distribution) + self.assertIn("AllowedMethods: [GET, HEAD, OPTIONS]", distribution) + self.assertIn("PathPattern: /api/*", distribution) + self.assertIn( + "AllowedMethods: [GET, HEAD, OPTIONS, PUT, PATCH, POST, DELETE]", + distribution, + ) + self.assertIn("OriginProtocolPolicy: http-only", distribution) + self.assertIn("HeaderName: X-FIP-Origin-Verify", distribution) + self.assertIn("HeaderValue: !Ref OriginVerificationToken", distribution) + + def test_caching_authorization_query_and_host_contract(self): + distribution = _block(self.template, "CloudFrontDistribution") + default_origin = _block(self.template, "DefaultOriginRequestPolicy") + self.assertEqual(distribution.count(f"CachePolicyId: {MANAGED_CACHING_DISABLED}"), 2) + self.assertIn( + f"OriginRequestPolicyId: {MANAGED_ALL_VIEWER_EXCEPT_HOST}", + distribution, + ) + self.assertIn("QueryStringBehavior: all", default_origin) + self.assertIn("CookieBehavior: none", default_origin) + self.assertIn("HeaderBehavior: none", default_origin) + self.assertNotRegex(default_origin, r"(?im)^\s*- Host\s*$") + + def test_no_invalid_custom_disabled_cache_policy_regression(self): + self.assertNotIn("DisabledDefaultCachePolicy:", self.template) + self.assertNotIn("DisabledApiCachePolicy:", self.template) + self.assertNotIn("Type: AWS::CloudFront::CachePolicy", self.template) + self.assertNotRegex( + self.template, + r"(?s)DefaultTTL:\s*0.*ParametersInCacheKeyAndForwardedToOrigin", + ) + + def test_origin_verification_token_is_noecho_and_only_placeholder_is_committed(self): + token = _block(self.template, "OriginVerificationToken") + self.assertIn("NoEcho: true", token) + self.assertIn("MinLength: 43", token) + values = {item["ParameterKey"]: item["ParameterValue"] for item in self.parameters} + self.assertEqual( + values["OriginVerificationToken"], + "REPLACE_WITH_43_PLUS_BASE64URL_RANDOM_VALUE", + ) + + def test_cognito_managed_login_mfa_and_public_pkce_client(self): + pool = _block(self.template, "CognitoUserPool") + client = _block(self.template, "CognitoUserPoolClient") + domain = _block(self.template, "CognitoUserPoolDomain") + branding = _block(self.template, "CognitoManagedLoginBranding") + self.assertIn("AllowAdminCreateUserOnly: true", pool) + self.assertIn("MfaConfiguration: 'ON'", pool) + self.assertIn("EnabledMfas: [SOFTWARE_TOKEN_MFA]", pool) + self.assertNotIn("SMS_MFA", pool) + self.assertNotIn("SoftwareTokenMfaConfiguration", pool) + self.assertIn("MinimumLength: 12", pool) + self.assertIn("GenerateSecret: false", client) + self.assertIn("AllowedOAuthFlows: [code]", client) + self.assertNotIn("implicit", client.lower()) + self.assertIn("${CloudFrontDistribution.DomainName}/auth/callback", client) + self.assertIn("${CloudFrontDistribution.DomainName}/'", client) + self.assertIn("ManagedLoginVersion: 2", domain) + self.assertIn("UseCognitoProvidedValues: true", branding) + + def test_cognito_deletion_protection_follows_deployment_state(self): + pool = _block(self.template, "CognitoUserPool") + self.assertIn( + "DeletionProtection: !If [OriginLockdown, ACTIVE, INACTIVE]", + pool, + ) + self.assertEqual( + "false", + next( + item["ParameterValue"] + for item in self.parameters + if item["ParameterKey"] == "OriginLockdownEnabled" + ), + ) + + def test_three_groups_and_least_privilege_task_role(self): + for logical_id, group in ( + ("CustomerGroup", "customer"), + ("OperatorGroup", "operator"), + ("AdminGroup", "admin"), + ): + self.assertIn(f"GroupName: {group}", _block(self.template, logical_id)) + task_role = _block(self.template, "ApplicationTaskRole") + self.assertIn("cognito-idp:AdminGetUser", task_role) + self.assertIn("cognito-idp:ListUsersInGroup", task_role) + self.assertNotIn("cognito-idp:*", task_role) + self.assertIn("Resource: !GetAtt CognitoUserPool.Arn", task_role) + + def test_backend_and_nginx_runtime_contract(self): + task = _block(self.template, "ApplicationTaskDefinition") + self.assertIn("Value: cognito_rbac", task) + self.assertIn("Name: COGNITO_USER_POOL_ID", task) + self.assertIn("Name: COGNITO_CLIENT_ID", task) + self.assertIn("proxy_pass http://127.0.0.1:8000;", self.nginx) + self.assertNotIn("ssl_certificate", self.nginx) + + def test_runtime_reader_writer_and_migration_secrets_are_separated(self): + application = _block(self.template, "ApplicationTaskDefinition") + application_execution = _block(self.template, "EcsExecutionRole") + migration = _block(self.template, "MigrationTaskDefinition") + migration_execution = _block(self.template, "MigrationExecutionRole") + self.assertIn("Name: DATABASE_USER", application) + self.assertIn("Name: DATABASE_WRITE_USER", application) + self.assertIn("Name: DATABASE_WRITE_PASSWORD", application) + self.assertNotIn("DATABASE_ADMIN", application) + self.assertNotIn("MasterUserSecret", application_execution) + self.assertIn("ExecutionRoleArn: !GetAtt MigrationExecutionRole.Arn", migration) + self.assertIn("MasterUserSecret", migration_execution) + + def test_worker_sidecar_is_private_and_uses_backend_runtime(self): + application = _block(self.template, "ApplicationTaskDefinition") + service = _block(self.template, "ApplicationService") + worker = application.split("- Name: worker", 1)[1] + self.assertIn("Image: !Ref BackendImageUri", worker) + self.assertIn("EntryPoint: [python, -m]", worker) + self.assertIn("Command: [pipelines.worker]", worker) + self.assertIn("Name: DATABASE_PIPELINE_PASSWORD", worker) + self.assertIn("Name: PIPELINE_SNAPSHOT_S3_URI", worker) + self.assertNotIn("PortMappings:", worker) + self.assertNotIn("ContainerName: worker", service) + self.assertIn("ContainerName: frontend", service) + self.assertIn("Cpu: !If [PipelineWorkerEnabled, '1024', '512']", application) + self.assertIn("Memory: !If [PipelineWorkerEnabled, '4096', '2048']", application) + + def test_pipeline_publisher_secret_and_snapshot_permissions_are_scoped(self): + execution = _block(self.template, "EcsExecutionRole") + task_role = _block(self.template, "ApplicationTaskRole") + prerequisite = _block(self.db_prerequisite, "DbAccessTaskDefinition") + self.assertIn("ApplicationDatabasePipelineSecretArn", execution) + self.assertIn("DiscoverCurrentSnapshotFallback", task_role) + self.assertIn("Action: s3:ListBucket", task_role) + self.assertIn("s3:prefix: pipeline-snapshots/current.db", task_role) + self.assertIn("s3:GetObject", task_role) + self.assertIn("s3:PutObject", task_role) + self.assertIn("pipeline-snapshots/current.db", task_role) + self.assertNotIn("s3:*", task_role) + self.assertIn("DATABASE_PIPELINE_PASSWORD", prerequisite) + + def test_external_api_secrets_are_injected_only_where_needed(self): + parameters = self.template.split("\nRules:\n", 1)[0] + application = _block(self.template, "ApplicationTaskDefinition") + execution = _block(self.template, "EcsExecutionRole") + backend, worker = application.split("- Name: worker", 1) + self.assertIn("AnthropicSecretArn", parameters) + self.assertIn("CharityCommissionSecretArn", parameters) + self.assertIn("Name: ANTHROPIC_API_KEY", backend) + self.assertIn("${AnthropicSecretArn}:api_key::", backend) + self.assertNotIn("CHARITY_COMMISSION_API_KEY", backend) + self.assertIn("Name: CHARITY_COMMISSION_API_KEY", worker) + self.assertIn("${CharityCommissionSecretArn}:api_key::", worker) + self.assertNotIn("ANTHROPIC_API_KEY", worker) + self.assertIn("!Ref AnthropicSecretArn", execution) + self.assertIn("!Ref CharityCommissionSecretArn", execution) + self.assertNotIn("Resource: '*'", execution) + self.assertNotIn("kms:Decrypt", execution) + + def test_db_prerequisite_stack_has_no_application_cutover_or_edge_resources(self): + source = self.db_prerequisite + self.assertIn("Type: AWS::SecretsManager::Secret", source) + self.assertIn("Type: AWS::ECS::TaskDefinition", source) + self.assertIn("Command: [migration.database_access]", source) + self.assertIn("CpuArchitecture: ARM64", source) + self.assertIn("User: '10001:10001'", source) + for forbidden in ( + "AWS::ECS::Service", + "AWS::RDS::DBInstance", + "AWS::CloudFront::Distribution", + "AWS::Cognito::", + "AWS::ElasticLoadBalancingV2::", + "AWS::EC2::VPC", + ): + self.assertNotIn(forbidden, source) + + def test_db_prerequisite_task_has_release_secrets_but_no_secret_outputs(self): + task = _block(self.db_prerequisite, "DbAccessTaskDefinition") + outputs = self.db_prerequisite.split("\nOutputs:\n", 1)[1] + self.assertIn("DATABASE_ADMIN_PASSWORD", task) + self.assertIn("DATABASE_READER_PASSWORD", task) + self.assertIn("DATABASE_WRITER_PASSWORD", task) + self.assertIn("DATABASE_PIPELINE_PASSWORD", task) + self.assertIn("WriterDatabaseSecretArn", outputs) + self.assertIn("PipelineDatabaseSecretArn", outputs) + self.assertNotIn("password", outputs.lower()) + + def test_outputs_do_not_expose_origin_token(self): + outputs = self.template.split("\nOutputs:\n", 1)[1] + self.assertIn("https://${CloudFrontDistribution.DomainName}", outputs) + self.assertNotIn("OriginVerificationToken", outputs) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_cognito_rbac.py b/src/tests/test_cognito_rbac.py new file mode 100644 index 0000000..93d4229 --- /dev/null +++ b/src/tests/test_cognito_rbac.py @@ -0,0 +1,275 @@ +import dataclasses +from datetime import datetime, timedelta, timezone +import json +import unittest + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from fastapi import Depends, FastAPI +from fastapi.testclient import TestClient +from jose import jwt +from jose.utils import base64url_encode + +from bff.config import SECURITY_SETTINGS, validate_security_settings +from bff.database import WriterDatabaseUnavailable +from bff.postgres import scraper_routes +from bff.postgres.dependencies import reader_sessions, writer_sessions +from bff.security import Role, SlidingWindowRateLimiter, require_roles + + +def _encoded_integer(value: int) -> str: + raw = value.to_bytes((value.bit_length() + 7) // 8, "big") + return base64url_encode(raw).decode("ascii") + + +class TestCognitoRbac(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + cls.other_private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + public_numbers = cls.private_key.public_key().public_numbers() + cls.private_pem = cls.private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + cls.other_private_pem = cls.other_private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + cls.issuer = "https://cognito-idp.eu-west-1.amazonaws.com/eu-west-1_testpool" + cls.settings = dataclasses.replace( + SECURITY_SETTINGS, + app_env="demo", + auth_mode="cognito_rbac", + dev_auth_enabled=False, + cognito_region="eu-west-1", + cognito_user_pool_id="eu-west-1_testpool", + cognito_client_id="test-public-client", + cognito_domain="https://fip-test.auth.eu-west-1.amazoncognito.com", + oidc_jwks_json=json.dumps( + { + "keys": [ + { + "kty": "RSA", + "kid": "cognito-test-key", + "use": "sig", + "alg": "RS256", + "n": _encoded_integer(public_numbers.n), + "e": _encoded_integer(public_numbers.e), + } + ] + } + ), + oidc_algorithms=("RS256",), + ) + validate_security_settings(cls.settings) + application = FastAPI() + application.state.security_settings = cls.settings + application.state.rate_limiter = SlidingWindowRateLimiter(10_000, 60) + + class FakeDatabase: + def __init__(self): + self.reader_calls = 0 + self.writer_calls = 0 + self.writer_available = True + + def sessions(self): + self.reader_calls += 1 + return "reader" + + def write_sessions(self): + self.writer_calls += 1 + if not self.writer_available: + raise WriterDatabaseUnavailable("writer unavailable in test") + return "writer" + + cls.database = FakeDatabase() + application.state.database = cls.database + + @application.get("/normal/read", dependencies=[Depends(require_roles(Role.CUSTOMER))]) + async def read(): + return {"ok": True} + + @application.post("/api/scraper/run", dependencies=[Depends(require_roles(Role.OPERATOR))]) + async def scraper_run(): + return {"ok": True} + + @application.post( + "/api/admin/pipeline/trigger", + dependencies=[Depends(require_roles(Role.OPERATOR))], + ) + async def pipeline_trigger(): + return {"ok": True} + + @application.get("/api/admin/guard", dependencies=[Depends(require_roles(Role.ADMIN))]) + async def admin(): + return {"ok": True} + + @application.get("/database/read", dependencies=[Depends(require_roles(Role.CUSTOMER))]) + async def database_read(sessions=Depends(reader_sessions)): + return {"pool": sessions} + + @application.post( + "/database/write", + dependencies=[Depends(require_roles(Role.OPERATOR))], + ) + async def database_write(sessions=Depends(writer_sessions)): + return {"pool": sessions} + + class FakeJobs: + async def latest_status(self): + return { + "status": "success", + "started_at": "2026-08-12T10:00:00+00:00", + "finished_at": "2026-08-12T10:05:00+00:00", + } + + class FakePipelines: + async def public_source_statuses(self): + return [ + { + "name": "approved-source", + "enabled": True, + "freshness_sla_hours": 24, + "last_success_at": "2026-08-12T10:05:00+00:00", + "record_count": 42, + } + ] + + application.include_router(scraper_routes.router) + application.dependency_overrides[scraper_routes._jobs] = FakeJobs + application.dependency_overrides[scraper_routes._pipelines] = FakePipelines + + cls.client = TestClient(application) + + def token( + self, + groups=("customer",), + *, + issuer=None, + client_id="test-public-client", + token_use="access", + expires_delta=timedelta(minutes=5), + kid="cognito-test-key", + private_pem=None, + ) -> str: + now = datetime.now(timezone.utc) + return jwt.encode( + { + "sub": "cognito-subject", + "username": "opaque-cognito-username", + "cognito:groups": list(groups), + "iss": issuer or self.issuer, + "client_id": client_id, + "token_use": token_use, + "iat": now, + "exp": now + expires_delta, + }, + private_pem or self.private_pem, + algorithm="RS256", + headers={"kid": kid}, + ) + + def request(self, path: str, token: str | None, method="GET"): + headers = {"Authorization": f"Bearer {token}"} if token else {} + return self.client.request(method, path, headers=headers) + + def test_valid_access_token_and_role_permissions(self): + customer = self.token() + self.assertEqual(self.request("/normal/read", customer).status_code, 200) + status_response = self.request("/api/scraper/status", customer) + self.assertEqual(status_response.status_code, 200) + self.assertEqual(status_response.json()["sources"][0]["record_count"], 42) + self.assertEqual(self.request("/api/scraper/run", customer, "POST").status_code, 403) + self.assertEqual( + self.request("/api/admin/pipeline/trigger", customer, "POST").status_code, + 403, + ) + self.assertEqual(self.request("/api/admin/guard", customer).status_code, 403) + operator = self.token(("operator",)) + self.assertEqual(self.request("/normal/read", operator).status_code, 200) + self.assertEqual(self.request("/api/scraper/status", operator).status_code, 200) + self.assertEqual(self.request("/api/scraper/run", operator, "POST").status_code, 200) + self.assertEqual( + self.request("/api/admin/pipeline/trigger", operator, "POST").status_code, + 200, + ) + self.assertEqual(self.request("/api/admin/guard", operator).status_code, 403) + admin = self.token(("admin",)) + self.assertEqual(self.request("/normal/read", admin).status_code, 200) + self.assertEqual(self.request("/api/scraper/status", admin).status_code, 200) + self.assertEqual(self.request("/api/scraper/run", admin, "POST").status_code, 200) + self.assertEqual(self.request("/api/admin/guard", admin).status_code, 200) + + def test_invalid_access_tokens_fail_with_401(self): + cases = { + "expired": self.token(expires_delta=timedelta(seconds=-1)), + "wrong_signature": self.token(private_pem=self.other_private_pem), + "unknown_kid": self.token(kid="unknown"), + "wrong_issuer": self.token(issuer="https://identity.example.invalid"), + "wrong_client": self.token(client_id="another-client"), + "id_token": self.token(token_use="id"), + } + valid = self.token() + cases["tampered"] = valid[:-2] + ("aa" if valid[-2:] != "aa" else "bb") + for name, token in cases.items(): + with self.subTest(name=name): + self.assertEqual(self.request("/normal/read", token).status_code, 401) + self.assertEqual(self.request("/normal/read", None).status_code, 401) + + def test_zero_or_multiple_app_groups_fail_with_403(self): + memberships = ( + (), + ("unrelated",), + ("customer", "operator"), + ("customer", "admin"), + ("operator", "admin"), + ("customer", "operator", "admin"), + ) + for groups in memberships: + with self.subTest(groups=groups): + self.assertEqual( + self.request("/normal/read", self.token(groups)).status_code, + 403, + ) + + def test_customer_never_resolves_writer_and_operator_does(self): + self.database.reader_calls = 0 + self.database.writer_calls = 0 + customer = self.token(("customer",)) + operator = self.token(("operator",)) + read = self.request("/database/read", customer) + self.assertEqual(read.status_code, 200) + self.assertEqual(read.json(), {"pool": "reader"}) + denied = self.request("/database/write", customer, "POST") + self.assertEqual(denied.status_code, 403) + self.assertEqual(self.database.writer_calls, 0) + allowed = self.request("/database/write", operator, "POST") + self.assertEqual(allowed.status_code, 200) + self.assertEqual(allowed.json(), {"pool": "writer"}) + self.assertEqual(self.database.writer_calls, 1) + + def test_writer_outage_preserves_customer_reads_and_fails_mutation_cleanly(self): + self.database.reader_calls = 0 + self.database.writer_calls = 0 + self.database.writer_available = False + try: + customer = self.token(("customer",)) + operator = self.token(("operator",)) + read = self.request("/database/read", customer) + self.assertEqual(read.status_code, 200) + self.assertEqual(read.json(), {"pool": "reader"}) + mutation = self.request("/database/write", operator, "POST") + self.assertEqual(mutation.status_code, 503) + self.assertEqual( + mutation.json()["detail"], + "The mutation database path is temporarily unavailable.", + ) + finally: + self.database.writer_available = True + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_database.py b/src/tests/test_database.py index f631936..806a7e2 100644 --- a/src/tests/test_database.py +++ b/src/tests/test_database.py @@ -6,7 +6,12 @@ from fastapi.testclient import TestClient from sqlalchemy.engine import make_url -from bff.database import DatabaseConfigurationError, DatabaseSettings +from bff.database import ( + DatabaseAccess, + DatabaseConfigurationError, + DatabaseSettings, + WriterDatabaseUnavailable, +) from bff.main import app @@ -59,6 +64,46 @@ def pool_status(): class TestDatabaseFoundation(unittest.TestCase): + def test_writer_credentials_never_fall_back_to_reader_identity(self): + environment = { + "DATABASE_HOST": "postgres", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_app", + "DATABASE_PASSWORD": "reader-secret", + } + access = DatabaseAccess.from_env(environment) + self.assertFalse(access.writer_configured) + with self.assertRaises(WriterDatabaseUnavailable): + access.write_sessions() + + def test_writer_inherits_only_non_secret_connection_settings(self): + settings = DatabaseSettings.writer_from_env( + { + "DATABASE_HOST": "postgres", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_app", + "DATABASE_PASSWORD": "reader-secret", + "DATABASE_WRITE_USER": "foundation_app_writer", + "DATABASE_WRITE_PASSWORD": "writer-secret", + } + ) + self.assertIsNotNone(settings) + assert settings is not None + self.assertEqual(settings.host, "postgres") + self.assertEqual(settings.user, "foundation_app_writer") + self.assertEqual(settings.password, "writer-secret") + self.assertNotEqual(settings.password, "reader-secret") + + def test_partial_writer_configuration_fails_closed(self): + with self.assertRaises(DatabaseConfigurationError): + DatabaseSettings.writer_from_env( + { + "DATABASE_HOST": "postgres", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_WRITE_USER": "foundation_app_writer", + } + ) + def test_secret_file_builds_encoded_async_postgresql_url(self): descriptor, password_path = tempfile.mkstemp() try: diff --git a/src/tests/test_database_access.py b/src/tests/test_database_access.py new file mode 100644 index 0000000..2a50434 --- /dev/null +++ b/src/tests/test_database_access.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import unittest + +from migration.database_access import ( + READER_TABLES, + WRITER_COLUMN_PRIVILEGES, + WRITER_READ_TABLES, + WRITER_SEQUENCES, + WRITER_TABLE_PRIVILEGES, + DatabaseAccessConfigurationError, + configure_reader_role, + configure_writer_role, + default_integrity_evidence, + identifier, +) + + +class _RecordingConnection: + def __init__( + self, + *, + role_exists: bool = False, + column_grants: list[dict[str, object]] | None = None, + memberships: list[dict[str, object]] | None = None, + role_attributes: dict[str, object] | None = None, + ): + self.role_exists = role_exists + self.column_grants = column_grants or [] + self.memberships = memberships or [] + self.role_attributes = role_attributes or { + "rolname": "runtime", + "rolsuper": False, + "rolcreaterole": False, + "rolcreatedb": False, + "rolreplication": False, + "rolbypassrls": False, + "rolinherit": False, + "rolcanlogin": True, + } + self.statements: list[str] = [] + + async def fetchval(self, statement: str, *_args): + self.statements.append(statement) + return self.role_exists + + async def fetchrow(self, statement: str, *_args): + self.statements.append(statement) + if not self.role_exists: + return None + return dict(self.role_attributes) + + async def execute(self, statement: str, *_args): + self.statements.append(statement) + if statement.startswith("CREATE ROLE"): + self.role_exists = True + self.role_attributes["rolinherit"] = False + self.role_attributes["rolcanlogin"] = True + if statement.startswith("ALTER ROLE") and "LOGIN NOINHERIT" in statement: + self.role_attributes["rolinherit"] = False + self.role_attributes["rolcanlogin"] = True + return "INSERT 0 1" + + async def fetch(self, statement: str, *_args): + self.statements.append(statement) + if "information_schema.column_privileges" in statement: + return self.column_grants + if "pg_auth_members" in statement: + return self.memberships + return [] + + +ENVIRONMENT = { + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_ADMIN_USER": "foundation_admin", + "DATABASE_READER_USER": "foundation_app", + "DATABASE_READER_PASSWORD": "reader-secret", + "DATABASE_WRITER_USER": "foundation_app_writer", + "DATABASE_WRITER_PASSWORD": "writer-secret", +} + + +class TestDatabaseAccessContract(unittest.IsolatedAsyncioTestCase): + async def test_reader_is_exact_select_allowlist_and_future_tables_fail_closed(self): + connection = _RecordingConnection() + await configure_reader_role(connection, ENVIRONMENT) # type: ignore[arg-type] + sql = "\n".join(connection.statements) + self.assertIn("default_transaction_read_only = on", sql) + self.assertIn("GRANT SELECT ON TABLE", sql) + for table in READER_TABLES: + self.assertIn(f'"{table}"', sql) + self.assertNotIn("GRANT SELECT ON ALL TABLES", sql) + self.assertNotIn("GRANT ALL", sql) + self.assertNotIn("GRANT INSERT", sql) + + def test_release_gate_relations_are_reader_allowlisted(self): + self.assertTrue( + { + "alembic_version", + "data_quality_issues", + "dataset_versions", + "job_runs", + "materialization_versions", + "migration_runs", + }.issubset(READER_TABLES) + ) + + async def test_writer_has_only_audited_table_and_column_dml(self): + connection = _RecordingConnection() + await configure_writer_role(connection, ENVIRONMENT) # type: ignore[arg-type] + sql = "\n".join(connection.statements) + self.assertIn("default_transaction_read_only = off", sql) + for table in WRITER_READ_TABLES: + self.assertIn(f'"{table}"', sql) + for table, privileges in WRITER_TABLE_PRIVILEGES.items(): + self.assertIn( + f'GRANT {", ".join(privileges)} ON TABLE "{table}"', + sql, + ) + for table, grants in WRITER_COLUMN_PRIVILEGES.items(): + for privilege, columns in grants.items(): + rendered = ", ".join(f'"{column}"' for column in columns) + self.assertIn( + f'GRANT {privilege} ({rendered}) ON TABLE "{table}"', + sql, + ) + self.assertEqual(WRITER_SEQUENCES, ()) + self.assertNotIn("GRANT ALL", sql) + self.assertNotIn("ON ALL TABLES", sql.replace("REVOKE ALL ON ALL TABLES", "")) + self.assertNotIn('UPDATE ON TABLE "charities"', sql) + self.assertNotIn('INSERT ON TABLE "retention_policies"', sql) + + async def test_existing_safe_writer_is_configured_without_superuser_attribute(self): + connection = _RecordingConnection( + role_exists=True, + column_grants=[ + { + "table_name": "source_configurations", + "privilege_type": "UPDATE", + "column_names": ["credentials_reference", "enabled"], + } + ], + ) + await configure_writer_role(connection, ENVIRONMENT) # type: ignore[arg-type] + sql = "\n".join(connection.statements) + self.assertIn( + 'ALTER ROLE "foundation_app_writer" LOGIN NOINHERIT PASSWORD', + sql, + ) + self.assertNotIn("SUPERUSER", sql) + self.assertIn( + 'REVOKE ALL ON ALL TABLES IN SCHEMA public FROM "foundation_app_writer"', + sql, + ) + self.assertIn( + 'REVOKE UPDATE ("credentials_reference", "enabled") ON TABLE ' + '"source_configurations" FROM "foundation_app_writer"', + sql, + ) + + async def test_existing_role_with_dangerous_attribute_fails_closed(self): + connection = _RecordingConnection( + role_exists=True, + role_attributes={ + "rolname": "foundation_app_writer", + "rolsuper": False, + "rolcreaterole": True, + "rolcreatedb": False, + "rolreplication": False, + "rolbypassrls": False, + "rolinherit": False, + "rolcanlogin": True, + }, + ) + with self.assertRaises(DatabaseAccessConfigurationError): + await configure_writer_role(connection, ENVIRONMENT) # type: ignore[arg-type] + self.assertFalse( + any(statement.startswith("ALTER ROLE") for statement in connection.statements) + ) + + async def test_existing_role_with_membership_fails_closed(self): + connection = _RecordingConnection( + role_exists=True, + memberships=[{"parent_role": "legacy_writer"}], + ) + with self.assertRaises(DatabaseAccessConfigurationError): + await configure_writer_role(connection, ENVIRONMENT) # type: ignore[arg-type] + self.assertFalse( + any(statement.startswith("ALTER ROLE") for statement in connection.statements) + ) + + def test_default_integrity_evidence_requires_preservation_and_only_defaults(self): + evidence = default_integrity_evidence( + before={ + "source_configurations": {"existing-source": "a" * 64}, + "retention_policies": {"existing-policy": "b" * 64}, + }, + after={ + "source_configurations": { + "existing-source": "a" * 64, + "default-source": "c" * 64, + }, + "retention_policies": { + "existing-policy": "b" * 64, + "default-policy": "d" * 64, + }, + }, + expected_default_ids={ + "source_configurations": ("default-source",), + "retention_policies": ("default-policy",), + }, + ) + self.assertTrue( + evidence["source_configurations"]["existing_checksums_unchanged"] + ) + self.assertTrue( + evidence["retention_policies"]["only_missing_defaults_added"] + ) + + def test_unsafe_or_shared_principal_input_is_rejected(self): + with self.assertRaises(DatabaseAccessConfigurationError): + identifier("foundation_app;drop", "DATABASE_READER_USER") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_database_access_integration.py b/src/tests/test_database_access_integration.py new file mode 100644 index 0000000..851f93f --- /dev/null +++ b/src/tests/test_database_access_integration.py @@ -0,0 +1,432 @@ +"""Explicit local PostgreSQL release gate for runtime database principals.""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +import subprocess +import sys +import unittest +import uuid + +import asyncpg + +from bff.database import DatabaseManager, DatabaseSettings +from bff.postgres.governance_repository import GovernanceRepository +from bff.postgres.job_repository import PostgresJobRepository +from bff.postgres.pipeline_repository import PipelineRepository +from migration.database_access import ( + READER_TABLES, + WRITER_READ_TABLES, + WRITER_TABLE_PRIVILEGES, + DatabaseAccessConfigurationError, + bootstrap_runtime_configuration, + configure_reader_role, + configure_writer_role, + connect_admin, + default_configuration_snapshot, + default_integrity_evidence, + grant_equality_evidence, + grant_snapshot, + role_security_evidence, + verify_reader_role, + verify_writer_role, +) +from governance.retention import load_governance_configuration +from pipelines.durable import load_source_configurations + + +ROOT = Path(__file__).resolve().parents[2] + + +class _ExecutingRecordingConnection: + """Record SQL while delegating every operation to a real PostgreSQL connection.""" + + def __init__(self, connection: asyncpg.Connection): + self.connection = connection + self.statements: list[str] = [] + + async def fetchrow(self, statement: str, *args): + self.statements.append(statement) + return await self.connection.fetchrow(statement, *args) + + async def fetchval(self, statement: str, *args): + self.statements.append(statement) + return await self.connection.fetchval(statement, *args) + + async def fetch(self, statement: str, *args): + self.statements.append(statement) + return await self.connection.fetch(statement, *args) + + async def execute(self, statement: str, *args): + self.statements.append(statement) + return await self.connection.execute(statement, *args) + + +@unittest.skipUnless( + os.getenv("RUN_DB_ACCESS_INTEGRATION") == "1", + "RUN_DB_ACCESS_INTEGRATION=1 is required", +) +class TestDatabaseAccessIntegration(unittest.TestCase): + def test_real_roles_startup_reads_and_representative_mutations(self): + asyncio.run(self._exercise()) + + @staticmethod + def _environment() -> dict[str, str]: + required = ( + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_NAME", + "DATABASE_ADMIN_USER", + "DATABASE_ADMIN_PASSWORD", + "DATABASE_READER_USER", + "DATABASE_READER_PASSWORD", + "DATABASE_WRITER_USER", + "DATABASE_WRITER_PASSWORD", + "DATABASE_CLUSTER_ADMIN_USER", + "DATABASE_CLUSTER_ADMIN_PASSWORD", + ) + environment = {name: os.environ.get(name, "") for name in required} + environment["DATABASE_SSL_MODE"] = os.environ.get( + "DATABASE_SSL_MODE", "disable" + ) + missing = [name for name in required if not environment[name]] + if missing: + raise AssertionError(f"Missing integration settings: {', '.join(missing)}") + if environment["DATABASE_HOST"] not in {"127.0.0.1", "localhost", "::1"}: + raise AssertionError("Database access integration is restricted to localhost") + return environment + + async def _exercise(self) -> None: + environment = self._environment() + admin = await connect_admin(environment) + try: + provisioning_attributes = await admin.fetchrow( + "SELECT rolsuper, rolcreaterole FROM pg_roles WHERE rolname=current_user" + ) + self.assertIsNotNone(provisioning_attributes) + self.assertFalse(provisioning_attributes["rolsuper"]) + self.assertTrue(provisioning_attributes["rolcreaterole"]) + has_reader_admin_option = await admin.fetchval( + """ + SELECT EXISTS ( + SELECT 1 + FROM pg_auth_members AS membership + JOIN pg_roles AS parent ON parent.oid=membership.roleid + JOIN pg_roles AS member ON member.oid=membership.member + WHERE parent.rolname=$1 AND member.rolname=current_user + AND membership.admin_option + ) + """, + environment["DATABASE_READER_USER"], + ) + self.assertTrue(has_reader_admin_option) + + await admin.execute( + """ + INSERT INTO dataset_versions ( + dataset_version, status, is_active, activated_at + ) VALUES ( + 'db-access-integration', 'active', TRUE, CURRENT_TIMESTAMP + ) + ON CONFLICT (dataset_version) DO NOTHING + """ + ) + await admin.execute( + """ + INSERT INTO materialization_versions ( + materialization_version_id, dataset_version, + materialization_name, revision, status, is_active, + row_count, activated_at + ) VALUES ( + '00000000-0000-4000-8000-000000000001', + 'db-access-integration', 'dashboard_analytics', 1, + 'active', TRUE, 0, CURRENT_TIMESTAMP + ) + ON CONFLICT ( + dataset_version, materialization_name, revision + ) DO NOTHING + """ + ) + await admin.execute( + """ + INSERT INTO analytics_scope_totals ( + dataset_version, amount_basis, currency, + total_grants, known_geography_grants, multi_country_grants, + invalid_amount_grants, missing_date_grants, + negative_amount_grants, zero_amount_grants, + classified_grants, unclassified_grants, + source_classified_grants, inferred_classified_grants, + multiple_programme_grants, invalid_source_label_grants, + low_confidence_grants, total_amount_minor + ) VALUES ( + 'db-access-integration', 'eur_converted', 'EUR', + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ) + ON CONFLICT (dataset_version, amount_basis, currency) DO NOTHING + """ + ) + + sources = tuple(load_source_configurations()) + governance = load_governance_configuration() + defaults_before = await default_configuration_snapshot(admin) + bootstrap = await bootstrap_runtime_configuration( + admin, sources=sources, governance=governance + ) + defaults_after = await default_configuration_snapshot(admin) + default_evidence = default_integrity_evidence( + before=defaults_before, + after=defaults_after, + expected_default_ids={ + "source_configurations": ( + source.source_name for source in sources + ), + "retention_policies": ( + policy.retention_class for policy in governance.policies + ), + }, + ) + + recording = _ExecutingRecordingConnection(admin) + await configure_reader_role(recording, environment) # type: ignore[arg-type] + await configure_writer_role(recording, environment) # type: ignore[arg-type] + # Exercise the existing-role path again with the same non-superuser. + await configure_reader_role(recording, environment) # type: ignore[arg-type] + await configure_writer_role(recording, environment) # type: ignore[arg-type] + role_statements = [ + statement + for statement in recording.statements + if statement.lstrip().upper().startswith(("ALTER ROLE", "CREATE ROLE")) + ] + self.assertTrue(role_statements) + self.assertTrue( + all("SUPERUSER" not in statement.upper() for statement in role_statements), + role_statements, + ) + snapshot = await grant_snapshot( + admin, + ( + environment["DATABASE_READER_USER"], + environment["DATABASE_WRITER_USER"], + ), + ) + grant_equality = grant_equality_evidence( + snapshot, + reader=environment["DATABASE_READER_USER"], + writer=environment["DATABASE_WRITER_USER"], + database_name=environment["DATABASE_NAME"], + ) + role_evidence = { + "reader": await role_security_evidence( + admin, + username=environment["DATABASE_READER_USER"], + database_name=environment["DATABASE_NAME"], + ), + "writer": await role_security_evidence( + admin, + username=environment["DATABASE_WRITER_USER"], + database_name=environment["DATABASE_NAME"], + ), + } + finally: + await admin.close() + + self.assertGreaterEqual(bootstrap["source_configurations_inserted"], 0) + self.assertGreaterEqual(bootstrap["retention_policies_inserted"], 0) + self.assertTrue( + default_evidence["source_configurations"]["existing_ids_preserved"] + ) + self.assertTrue( + default_evidence["retention_policies"]["existing_checksums_unchanged"] + ) + self.assertTrue( + grant_equality["all_grants_equal_allowlist"], grant_equality + ) + for evidence in role_evidence.values(): + self.assertFalse(evidence["rolsuper"]) + self.assertFalse(evidence["rolcreaterole"]) + self.assertFalse(evidence["rolcreatedb"]) + self.assertFalse(evidence["rolreplication"]) + self.assertFalse(evidence["rolbypassrls"]) + self.assertEqual(evidence["memberships"], []) + self.assertEqual(sum(evidence["owned_objects"].values()), 0) + + reader_result = await verify_reader_role(environment) + writer_result = await verify_writer_role(environment) + for check, passed in reader_result.items(): + if check == "tls_in_use" and environment["DATABASE_SSL_MODE"] == "disable": + self.assertFalse(passed) + else: + self.assertTrue(passed, check) + self.assertTrue(all(writer_result.values()), writer_result) + + table_grants = { + (row["grantee"], row["table_name"], row["privilege_type"]) + for row in snapshot["table_privileges"] + } + reader = environment["DATABASE_READER_USER"] + writer = environment["DATABASE_WRITER_USER"] + expected_reader = {(reader, table, "SELECT") for table in READER_TABLES} + expected_writer = {(writer, table, "SELECT") for table in WRITER_READ_TABLES} + expected_writer.update( + (writer, table, privilege) + for table, privileges in WRITER_TABLE_PRIVILEGES.items() + for privilege in privileges + ) + self.assertEqual( + {grant for grant in table_grants if grant[0] == reader}, expected_reader + ) + self.assertEqual( + {grant for grant in table_grants if grant[0] == writer}, expected_writer + ) + self.assertEqual(snapshot["sequence_privileges"], []) + + writer_settings = DatabaseSettings.writer_from_env( + { + "DATABASE_HOST": environment["DATABASE_HOST"], + "DATABASE_PORT": environment["DATABASE_PORT"], + "DATABASE_NAME": environment["DATABASE_NAME"], + "DATABASE_SSL_MODE": environment["DATABASE_SSL_MODE"], + "DATABASE_WRITE_USER": environment["DATABASE_WRITER_USER"], + "DATABASE_WRITE_PASSWORD": environment["DATABASE_WRITER_PASSWORD"], + } + ) + self.assertIsNotNone(writer_settings) + writer_database = DatabaseManager(writer_settings) # type: ignore[arg-type] + try: + sessions = writer_database.sessions() + operator_job = await PostgresJobRepository(sessions).enqueue( + "refresh_charities", + {"mode": "refresh_charities"}, + actor_id="operator-integration", + idempotency_key=f"operator-{uuid.uuid4()}", + ) + self.assertEqual(operator_job["status"], "queued") + + governance = GovernanceRepository(sessions) + hold = await governance.create_hold( + hold_type="incident", + scope_type="dataset", + scope_id="db-access-integration", + reason="database access integration", + created_by="admin-integration", + ) + released = await governance.release_hold( + hold["data_hold_id"], + released_by="admin-integration", + release_reason="integration complete", + ) + self.assertEqual(released["status"], "released") + + source_name = load_source_configurations()[0].source_name + source = await PipelineRepository(sessions).set_source_enabled( + source_name, enabled=False + ) + self.assertFalse(source["enabled"]) + finally: + await writer_database.close() + + self._assert_reader_only_runtime(environment) + await self._assert_dangerous_role_fails_closed(environment) + + async def _assert_dangerous_role_fails_closed( + self, environment: dict[str, str] + ) -> None: + dangerous_role = f"fip_dangerous_{uuid.uuid4().hex[:16]}" + dangerous_password = uuid.uuid4().hex + uuid.uuid4().hex + cluster_admin = await asyncpg.connect( + host=environment["DATABASE_HOST"], + port=int(environment["DATABASE_PORT"]), + user=environment["DATABASE_CLUSTER_ADMIN_USER"], + password=environment["DATABASE_CLUSTER_ADMIN_PASSWORD"], + database=environment["DATABASE_NAME"], + ssl=environment["DATABASE_SSL_MODE"], + ) + try: + await cluster_admin.execute( + f'CREATE ROLE "{dangerous_role}" LOGIN CREATEDB' + ) + await cluster_admin.execute( + f'GRANT "{dangerous_role}" TO ' + f'"{environment["DATABASE_ADMIN_USER"]}" WITH ADMIN OPTION' + ) + release = await connect_admin(environment) + try: + recording = _ExecutingRecordingConnection(release) + dangerous_environment = dict(environment) + dangerous_environment["DATABASE_READER_USER"] = dangerous_role + dangerous_environment["DATABASE_READER_PASSWORD"] = dangerous_password + with self.assertRaises(DatabaseAccessConfigurationError): + await configure_reader_role( # type: ignore[arg-type] + recording, dangerous_environment + ) + self.assertFalse( + any( + statement.lstrip().upper().startswith("ALTER ROLE") + for statement in recording.statements + ) + ) + finally: + await release.close() + finally: + await cluster_admin.execute( + f'REVOKE "{dangerous_role}" FROM ' + f'"{environment["DATABASE_ADMIN_USER"]}"' + ) + await cluster_admin.execute(f'DROP ROLE "{dangerous_role}"') + await cluster_admin.close() + + def _assert_reader_only_runtime(self, environment: dict[str, str]) -> None: + script = """ +from fastapi.testclient import TestClient +from bff.main import app + +with TestClient(app) as client: + assert app.state.database.writer_configured is False + expected = { + '/health/ready': 200, + '/api/charities/stats': 200, + '/api/charities/grants/map': 200, + '/api/charities/directory/organizations': 200, + '/api/scraper/status': 200, + } + observed = {path: client.get(path).status_code for path in expected} + assert observed == expected, observed +""" + process_environment = { + key: value + for key, value in os.environ.items() + if not key.startswith("DATABASE_") + } + process_environment.update( + { + "APP_ENV": "demo", + "DATA_RUNTIME_MODE": "postgresql", + "AUTH_MODE": "public_readonly", + "DEV_AUTH_ENABLED": "false", + "CORE_PROXY_ENABLED": "false", + "CORS_ORIGINS": "", + "DATABASE_HOST": environment["DATABASE_HOST"], + "DATABASE_PORT": environment["DATABASE_PORT"], + "DATABASE_NAME": environment["DATABASE_NAME"], + "DATABASE_USER": environment["DATABASE_READER_USER"], + "DATABASE_PASSWORD": environment["DATABASE_READER_PASSWORD"], + "DATABASE_SSL_MODE": environment["DATABASE_SSL_MODE"], + "PYTHONPATH": str(ROOT / "src"), + } + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=ROOT, + env=process_environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_durable_pipeline.py b/src/tests/test_durable_pipeline.py index 3c584e6..4ad85d5 100644 --- a/src/tests/test_durable_pipeline.py +++ b/src/tests/test_durable_pipeline.py @@ -33,6 +33,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[2] MIGRATION_PATH = REPOSITORY_ROOT / "alembic" / "versions" / "0005_durable_pipeline.py" +WORKER_MIGRATION_PATH = REPOSITORY_ROOT / "alembic" / "versions" / "0007_worker_execution.py" REQUIRED_SOURCES = { "360giving", "charity_commission", @@ -227,6 +228,14 @@ def test_production_pipeline_code_has_no_local_lock_or_subprocess_coordination(s ): self.assertNotIn(forbidden, combined) + def test_worker_migration_adds_active_dedupe_without_rewriting_job_history(self): + source = WORKER_MIGRATION_PATH.read_text(encoding="utf-8") + self.assertIn("ALTER TABLE job_runs ADD COLUMN active_dedupe_key", source) + self.assertIn("CREATE UNIQUE INDEX uq_job_runs_active_dedupe", source) + self.assertIn("status IN ('queued', 'running')", source) + for forbidden in ("DROP TABLE JOB_RUNS", "TRUNCATE", "DELETE FROM JOB_RUNS"): + self.assertNotIn(forbidden, source.upper()) + @unittest.skipUnless( os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), diff --git a/src/tests/test_postgres_application.py b/src/tests/test_postgres_application.py index 3ac191a..aba2c2c 100644 --- a/src/tests/test_postgres_application.py +++ b/src/tests/test_postgres_application.py @@ -50,6 +50,49 @@ def _route_contract(router) -> set[tuple[str, tuple[str, ...]]]: class TestPostgreSQLRouteParity(unittest.TestCase): + def test_cognito_runtime_startup_performs_no_database_io(self): + script = """ +from fastapi.testclient import TestClient +from bff.main import app +with TestClient(app): + pass +""" + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith("DATABASE_") + } + environment.update( + { + "APP_ENV": "demo", + "DATA_RUNTIME_MODE": "postgresql", + "AUTH_MODE": "cognito_rbac", + "COGNITO_REGION": "eu-west-1", + "COGNITO_USER_POOL_ID": "eu-west-1_startupcheck", + "COGNITO_CLIENT_ID": "startup-check-client", + "COGNITO_DOMAIN": "https://startup-check.auth.eu-west-1.amazoncognito.com", + "CORS_ORIGINS": "", + "DEV_AUTH_ENABLED": "false", + "CORE_PROXY_ENABLED": "false", + "DATABASE_HOST": "database-must-not-be-contacted.invalid", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_app", + "DATABASE_PASSWORD": "not-a-real-secret", + "DATABASE_CONNECT_TIMEOUT_SECONDS": "1", + "PYTHONPATH": str(Path(__file__).resolve().parents[2] / "src"), + } + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[2], + env=environment, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + def test_postgresql_router_covers_every_legacy_application_route(self): from bff.charity import router as legacy_router from bff.postgres.routes import router as postgresql_router diff --git a/src/tests/test_profile_hydration_worker.py b/src/tests/test_profile_hydration_worker.py new file mode 100644 index 0000000..e79d106 --- /dev/null +++ b/src/tests/test_profile_hydration_worker.py @@ -0,0 +1,199 @@ +"""Focused contracts for durable source-funder profile hydration.""" + +from __future__ import annotations + +import asyncio +import os +import unittest +import uuid +from unittest.mock import AsyncMock, patch + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from bff.database import DatabaseSettings +from bff.postgres.funder_repository import SourceFunderRepository +from bff.postgres.job_repository import PostgresJobRepository +from pipelines.durable_worker import DurableWorker +from pipelines.worker_handlers import WorkerHandlers + + +class _FakeJobRepository: + def __init__(self, job: dict) -> None: + self.job = job + self.succeeded: list[tuple[str, dict]] = [] + self.failed: list[tuple[str, dict]] = [] + + async def claim(self, **_kwargs): + job, self.job = self.job, None + return job + + async def heartbeat(self, *_args, **_kwargs): + return True + + async def succeed(self, job_id, **kwargs): + self.succeeded.append((job_id, kwargs)) + return True + + async def fail(self, job_id, **kwargs): + self.failed.append((job_id, kwargs)) + return "failed" + + +class TestProfileHydrationWorker(unittest.IsolatedAsyncioTestCase): + async def test_hydration_job_is_registered_and_completes_without_unsupported_type(self): + job_id = str(uuid.uuid4()) + repository = _FakeJobRepository( + { + "job_id": job_id, + "job_type": "source_funder_profile_hydration", + "input": {"source_funder_key": "gb-chc:fixture"}, + "timeout_seconds": 5, + "attempt": 1, + } + ) + handlers = WorkerHandlers( + object(), + pipeline_settings=None, + artifact_store=None, + source_schema_version=None, + code_revision=None, + ) + hydrated = AsyncMock( + return_value={ + "source_funder_key": "gb-chc:fixture", + "profile_id": 123, + "status": "ready", + } + ) + + with patch( + "pipelines.worker_handlers.SourceFunderRepository" + ) as repository_type: + repository_type.return_value.hydrate_profile_cache = hydrated + result = await DurableWorker( + repository, + handlers.mapping, + worker_id="worker-test", + ).run_once() + + self.assertIn("source_funder_profile_hydration", handlers.mapping) + self.assertEqual(result.status, "succeeded") + self.assertFalse(repository.failed) + hydrated.assert_awaited_once_with( + "gb-chc:fixture", + job_id=job_id, + ) + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "RUN_POSTGRES_INTEGRATION=1 or TEST_DATABASE_URL is required", +) +class TestProfileHydrationPostgreSQLIntegration(unittest.TestCase): + def test_success_and_terminal_failure_update_profile_cache(self): + asyncio.run(self._exercise()) + + @staticmethod + def _database_url() -> str: + return os.getenv("TEST_DATABASE_URL") or DatabaseSettings.from_env().sqlalchemy_url() + + async def _exercise(self) -> None: + engine = create_async_engine(self._database_url(), pool_pre_ping=True) + connection = await engine.connect() + outer_transaction = await connection.begin() + sessions = async_sessionmaker( + bind=connection, + expire_on_commit=False, + autoflush=False, + join_transaction_mode="create_savepoint", + ) + try: + async with sessions() as session: + identity = ( + await session.execute( + text( + """ + SELECT fact.source_funder_key, fact.linked_profile_id + FROM grant_source_funder_facts AS fact + JOIN dataset_versions AS dataset + ON dataset.dataset_version=fact.dataset_version + AND dataset.is_active + JOIN charities AS profile + ON profile.dataset_version=fact.dataset_version + AND profile.charity_id=fact.linked_profile_id + WHERE fact.linked_profile_id IS NOT NULL + ORDER BY fact.source_funder_key LIMIT 1 + """ + ) + ) + ).one() + key = str(identity[0]) + funders = SourceFunderRepository(sessions) + handlers = WorkerHandlers( + sessions, + pipeline_settings=None, + artifact_store=None, + source_schema_version=None, + code_revision=None, + ) + + queued = await funders.queue_profile_cache( + key, + actor_id="profile-hydration-test", + idempotency_key=f"profile-success-{uuid.uuid4()}", + ) + result = await handlers.source_funder_profile_hydration( + { + "job_id": queued["job_id"], + "job_type": "source_funder_profile_hydration", + "input": {"source_funder_key": key}, + } + ) + cached = await funders.profile_cache(key) + self.assertEqual(result["accepted_count"], 1) + self.assertEqual(cached["status"], "ready") + self.assertIsInstance(cached["payload"], dict) + self.assertIn("all_details", cached["payload"]) + + failed_job = await funders.queue_profile_cache( + key, + actor_id="profile-hydration-test", + idempotency_key=f"profile-failure-{uuid.uuid4()}", + ) + async with sessions() as session, session.begin(): + await session.execute( + text( + """ + UPDATE job_runs SET status='running', + started_at=CURRENT_TIMESTAMP, + heartbeat_at=CURRENT_TIMESTAMP + WHERE job_run_id=:job_id + """ + ), + {"job_id": uuid.UUID(failed_job["job_id"])}, + ) + status = await PostgresJobRepository(sessions).fail( + failed_job["job_id"], + worker_id="profile-hydration-test", + error_class="UnexpectedHandlerError", + failure_reason="redacted worker failure", + retryable=False, + ) + failed_cache = await funders.profile_cache(key) + self.assertEqual(status, "failed") + self.assertEqual(failed_cache["status"], "failed") + self.assertIsNone(failed_cache["payload"]) + self.assertEqual( + failed_cache["error"], + "Profile hydration could not be completed.", + ) + finally: + if outer_transaction.is_active: + await outer_transaction.rollback() + await connection.close() + await engine.dispose() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_release_gate.py b/src/tests/test_release_gate.py index 79f7d57..5ded90f 100644 --- a/src/tests/test_release_gate.py +++ b/src/tests/test_release_gate.py @@ -7,7 +7,7 @@ class TestReleaseGate(unittest.TestCase): def test_approved_active_dataset_passes(self): result = evaluate_release_state( { - "schema_version": "0006_governance_retention", + "schema_version": "0007_worker_execution", "active_dataset_count": 1, "dataset_version": "dataset-approved", "migration_status": "active", diff --git a/src/tests/test_security.py b/src/tests/test_security.py index 07e60c8..10ff511 100644 --- a/src/tests/test_security.py +++ b/src/tests/test_security.py @@ -309,6 +309,7 @@ def test_public_readonly_allows_only_reviewed_ui_read_routes(self): "/api/charities/grants/summary", "/api/charities/grants/themes", "/api/charities/grants/trends", + "/api/scraper/status", }, ) self.assertEqual(PUBLIC_READONLY_METHOD_ALLOWLIST, {"GET", "HEAD"}) diff --git a/src/tests/test_user_management.py b/src/tests/test_user_management.py new file mode 100644 index 0000000..9b47b8b --- /dev/null +++ b/src/tests/test_user_management.py @@ -0,0 +1,279 @@ +import dataclasses +import unittest +from unittest.mock import AsyncMock, patch + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from bff.config import SECURITY_SETTINGS +from bff.security import IdempotencyStore, Principal, Role, SlidingWindowRateLimiter +from bff.user_management import router + + +class FakeCognitoError(Exception): + def __init__(self, code: str): + self.response = {"Error": {"Code": code}} + super().__init__(code) + + +class FakeCognitoClient: + def __init__(self): + self.users = { + "admin-one": { + "Username": "admin-one", + "Enabled": True, + "UserStatus": "CONFIRMED", + "Attributes": [ + {"Name": "sub", "Value": "sub-admin-one"}, + {"Name": "email", "Value": "admin1@example.test"}, + ], + }, + "admin-two": { + "Username": "admin-two", + "Enabled": True, + "UserStatus": "CONFIRMED", + "Attributes": [ + {"Name": "sub", "Value": "sub-admin-two"}, + {"Name": "email", "Value": "admin2@example.test"}, + ], + }, + "customer-one": { + "Username": "customer-one", + "Enabled": True, + "UserStatus": "CONFIRMED", + "Attributes": [ + {"Name": "sub", "Value": "sub-customer-one"}, + {"Name": "email", "Value": "customer@example.test"}, + ], + }, + } + self.groups = { + "admin-one": {"admin"}, + "admin-two": {"admin"}, + "customer-one": {"customer"}, + } + self.fail_list = False + + def list_users(self, **kwargs): + if self.fail_list: + raise FakeCognitoError("InternalErrorException") + names = sorted(self.users) + start = int(kwargs.get("PaginationToken", "0")) + limit = int(kwargs["Limit"]) + selected = names[start : start + limit] + response = {"Users": [self.users[name] for name in selected]} + if start + limit < len(names): + response["PaginationToken"] = str(start + limit) + return response + + def admin_list_groups_for_user(self, **kwargs): + groups = sorted(self.groups[kwargs["Username"]]) + start = int(kwargs.get("NextToken", "0")) + selected = groups[start : start + 1] + response = {"Groups": [{"GroupName": group} for group in selected]} + if start + 1 < len(groups): + response["NextToken"] = str(start + 1) + return response + + def admin_get_user(self, **kwargs): + username = kwargs["Username"] + if username not in self.users: + raise FakeCognitoError("UserNotFoundException") + return self.users[username] + + def admin_create_user(self, **kwargs): + email = kwargs["Username"] + if any( + attribute["Value"] == email + for user in self.users.values() + for attribute in user["Attributes"] + if attribute["Name"] == "email" + ): + raise FakeCognitoError("UsernameExistsException") + username = f"opaque-{len(self.users) + 1}" + user = { + "Username": username, + "Enabled": True, + "UserStatus": "FORCE_CHANGE_PASSWORD", + "Attributes": [ + {"Name": "sub", "Value": f"sub-{username}"}, + {"Name": "email", "Value": email}, + ], + } + self.users[username] = user + self.groups[username] = set() + return {"User": user} + + def admin_add_user_to_group(self, **kwargs): + self.groups[kwargs["Username"]].add(kwargs["GroupName"]) + + def admin_remove_user_from_group(self, **kwargs): + self.groups[kwargs["Username"]].discard(kwargs["GroupName"]) + + def list_users_in_group(self, **kwargs): + names = sorted( + name for name, groups in self.groups.items() if kwargs["GroupName"] in groups + ) + start = int(kwargs.get("NextToken", "0")) + selected = names[start : start + 1] + response = {"Users": [self.users[name] for name in selected]} + if start + 1 < len(names): + response["NextToken"] = str(start + 1) + return response + + def admin_disable_user(self, **kwargs): + self.users[kwargs["Username"]]["Enabled"] = False + + def admin_enable_user(self, **kwargs): + self.users[kwargs["Username"]]["Enabled"] = True + + def admin_reset_user_password(self, **kwargs): + self.users[kwargs["Username"]]["UserStatus"] = "RESET_REQUIRED" + + +class TestUserManagement(unittest.TestCase): + def setUp(self): + self.fake = FakeCognitoClient() + application = FastAPI() + application.include_router(router) + application.state.security_settings = dataclasses.replace( + SECURITY_SETTINGS, + app_env="demo", + auth_mode="cognito_rbac", + dev_auth_enabled=False, + cognito_region="eu-west-1", + cognito_user_pool_id="eu-west-1_testpool", + cognito_client_id="client", + cognito_domain="https://fip-test.auth.eu-west-1.amazoncognito.com", + ) + application.state.cognito_client = self.fake + application.state.rate_limiter = SlidingWindowRateLimiter(10_000, 60) + application.state.idempotency_store = IdempotencyStore() + self.client = TestClient(application) + + @staticmethod + def principal(role: Role, subject="admin-caller") -> Principal: + return Principal(subject, frozenset({role}), {"sub": subject}) + + def call(self, method, path, role=Role.ADMIN, subject="admin-caller", **kwargs): + with patch( + "bff.security.authenticate_request", + new=AsyncMock(return_value=self.principal(role, subject)), + ): + return self.client.request(method, path, **kwargs) + + @staticmethod + def mutation_headers(index: str): + return {"Idempotency-Key": f"user-management-{index}"} + + def test_pagination_next_token_and_minimal_attributes(self): + first = self.call("GET", "/api/admin/users?page_size=2") + self.assertEqual(first.status_code, 200) + self.assertEqual(len(first.json()["users"]), 2) + self.assertEqual(first.json()["next_token"], "2") + second = self.call("GET", "/api/admin/users?page_size=2&next_token=2") + self.assertEqual(second.status_code, 200) + self.assertEqual(len(second.json()["users"]), 1) + for user in first.json()["users"] + second.json()["users"]: + self.assertEqual(set(user), {"id", "email", "enabled", "status", "role"}) + + def test_multiple_application_groups_are_detected_across_pages(self): + self.fake.groups["customer-one"].add("operator") + response = self.call("GET", "/api/admin/users?page_size=3") + self.assertEqual(response.status_code, 200) + customer = next(user for user in response.json()["users"] if user["id"] == "customer-one") + self.assertIsNone(customer["role"]) + + def test_invite_permissions_and_duplicate_error(self): + payload = {"email": "new@example.test", "role": "customer"} + for role in (Role.CUSTOMER, Role.OPERATOR): + response = self.call( + "POST", + "/api/admin/users", + role=role, + json=payload, + headers=self.mutation_headers(role.value), + ) + self.assertEqual(response.status_code, 403) + created = self.call( + "POST", + "/api/admin/users", + json=payload, + headers=self.mutation_headers("create"), + ) + self.assertEqual(created.status_code, 201) + self.assertEqual(created.json()["role"], "customer") + duplicate = self.call( + "POST", + "/api/admin/users", + json=payload, + headers=self.mutation_headers("duplicate"), + ) + self.assertEqual(duplicate.status_code, 409) + + def test_invalid_role_and_provider_failure_are_safe(self): + invalid = self.call( + "POST", + "/api/admin/users", + json={"email": "new@example.test", "role": "superuser"}, + headers=self.mutation_headers("invalid"), + ) + self.assertEqual(invalid.status_code, 400) + self.fake.fail_list = True + failed = self.call("GET", "/api/admin/users") + self.assertEqual(failed.status_code, 502) + self.assertNotIn("InternalError", failed.text) + + def test_last_admin_and_self_protection(self): + self.fake.users["admin-two"]["Enabled"] = False + last_disable = self.call( + "POST", + "/api/admin/users/admin-one/disable", + headers=self.mutation_headers("last-disable"), + ) + self.assertEqual(last_disable.status_code, 409) + last_downgrade = self.call( + "PATCH", + "/api/admin/users/admin-one/role", + json={"role": "operator"}, + headers=self.mutation_headers("last-downgrade"), + ) + self.assertEqual(last_downgrade.status_code, 409) + + self.fake.users["admin-two"]["Enabled"] = True + self_disable = self.call( + "POST", + "/api/admin/users/admin-one/disable", + subject="sub-admin-one", + headers=self.mutation_headers("self-disable"), + ) + self.assertEqual(self_disable.status_code, 409) + self_downgrade = self.call( + "PATCH", + "/api/admin/users/admin-one/role", + subject="sub-admin-one", + json={"role": "operator"}, + headers=self.mutation_headers("self-downgrade"), + ) + self.assertEqual(self_downgrade.status_code, 409) + + def test_other_admin_can_be_disabled_or_downgraded_when_two_are_active(self): + disabled = self.call( + "POST", + "/api/admin/users/admin-two/disable", + headers=self.mutation_headers("other-disable"), + ) + self.assertEqual(disabled.status_code, 200) + self.fake.users["admin-two"]["Enabled"] = True + downgraded = self.call( + "PATCH", + "/api/admin/users/admin-two/role", + json={"role": "operator"}, + headers=self.mutation_headers("other-downgrade"), + ) + self.assertEqual(downgraded.status_code, 200) + self.assertEqual(downgraded.json()["role"], "operator") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_worker_execution.py b/src/tests/test_worker_execution.py new file mode 100644 index 0000000..c307f92 --- /dev/null +++ b/src/tests/test_worker_execution.py @@ -0,0 +1,187 @@ +"""Focused contracts for the long-running PostgreSQL worker execution path.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +import tempfile +import unittest + +from pipelines.durable_worker import DurableWorker +from pipelines.worker_handlers import ( + LocalPipelineArtifactStore, + WorkerConfigurationError, + sha256_file, +) + + +ROOT = Path(__file__).resolve().parents[2] + + +class FakeJobRepository: + def __init__(self, jobs: list[dict] | None = None) -> None: + self.jobs = list(jobs or []) + self.claims = 0 + self.heartbeats = 0 + self.succeeded: list[tuple[str, dict]] = [] + self.failed: list[tuple[str, dict]] = [] + + async def claim(self, **_kwargs): + self.claims += 1 + return self.jobs.pop(0) if self.jobs else None + + async def heartbeat(self, *_args, **_kwargs): + self.heartbeats += 1 + return True + + async def succeed(self, job_id, **kwargs): + self.succeeded.append((job_id, kwargs)) + return True + + async def fail(self, job_id, **kwargs): + self.failed.append((job_id, kwargs)) + return "failed" + + +def job(job_id: str, job_type: str = "fixture", timeout: int = 5) -> dict: + return { + "job_id": job_id, + "job_type": job_type, + "input": {"fixture": True}, + "timeout_seconds": timeout, + "attempt": 1, + } + + +class TestWorkerExecution(unittest.IsolatedAsyncioTestCase): + async def test_queued_job_dispatches_and_succeeds_only_after_handler(self): + repository = FakeJobRepository([job("job-success")]) + domain_change = asyncio.Event() + + async def handler(_job): + domain_change.set() + return {"accepted_count": 1} + + result = await DurableWorker( + repository, {"fixture": handler}, worker_id="worker-a" + ).run_once() + + self.assertTrue(domain_change.is_set()) + self.assertEqual(result.status, "succeeded") + self.assertEqual(repository.succeeded[0][0], "job-success") + self.assertFalse(repository.failed) + + async def test_handler_exception_fails_job_and_worker_processes_next_job(self): + repository = FakeJobRepository([job("job-failed"), job("job-next")]) + attempts = 0 + + async def handler(_job): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("password=must-not-survive") + return {"accepted_count": 1} + + worker = DurableWorker( + repository, {"fixture": handler}, worker_id="worker-a" + ) + failed = await worker.run_once() + succeeded = await worker.run_once() + + self.assertEqual(failed.status, "failed") + self.assertEqual(succeeded.status, "succeeded") + self.assertFalse(repository.failed[0][1]["retryable"]) + self.assertNotIn( + "must-not-survive", repository.failed[0][1]["failure_reason"] + ) + + async def test_unsupported_type_and_timeout_are_terminal_failures(self): + unsupported = FakeJobRepository([job("job-unsupported", "unknown")]) + result = await DurableWorker( + unsupported, {}, worker_id="worker-a" + ).run_once() + self.assertEqual(result.status, "failed") + self.assertEqual( + unsupported.failed[0][1]["error_class"], "UnsupportedJobType" + ) + + async def blocked(_job): + await asyncio.Event().wait() + return {} + + timed = FakeJobRepository([job("job-timeout", timeout=0.001)]) + result = await DurableWorker( + timed, {"fixture": blocked}, worker_id="worker-a" + ).run_once() + self.assertEqual(result.status, "failed") + self.assertEqual(timed.failed[0][1]["error_class"], "JobTimeout") + + async def test_idle_claim_returns_without_dispatch(self): + repository = FakeJobRepository() + result = await DurableWorker( + repository, {}, worker_id="worker-a" + ).run_once() + self.assertEqual(result.status, "idle") + self.assertEqual(repository.claims, 1) + + +class TestWorkerStorageAndContracts(unittest.TestCase): + def test_local_snapshot_store_verifies_and_atomically_reuses_snapshot(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fallback = root / "fallback.db" + current = root / "current.db" + destination = root / "work" / "charities.db" + fallback.write_bytes(b"verified-baseline") + fallback_checksum = sha256_file(fallback) + store = LocalPipelineArtifactStore(current, fallback, fallback_checksum) + + self.assertEqual(store.download_baseline(destination), fallback_checksum) + destination.write_bytes(b"new-snapshot") + new_checksum = sha256_file(destination) + store.publish_snapshot(destination, checksum=new_checksum, job_id="job-1") + destination.unlink() + self.assertEqual(store.download_baseline(destination), new_checksum) + + store.checksum_path.write_text("0" * 64, encoding="utf-8") + with self.assertRaisesRegex( + WorkerConfigurationError, "checksum verification" + ): + store.download_baseline(destination) + + def test_postgres_claim_and_active_dedupe_are_database_enforced(self): + repository = (ROOT / "src/bff/postgres/job_repository.py").read_text( + encoding="utf-8" + ) + migration = (ROOT / "alembic/versions/0007_worker_execution.py").read_text( + encoding="utf-8" + ) + self.assertIn("FOR UPDATE SKIP LOCKED", repository) + self.assertIn("status IN ('queued', 'running')", migration) + self.assertIn("CREATE UNIQUE INDEX uq_job_runs_active_dedupe", migration) + + def test_worker_loop_has_backoff_and_signal_shutdown(self): + source = (ROOT / "src/pipelines/worker.py").read_text(encoding="utf-8") + self.assertIn("asyncio.wait_for(stop.wait(), timeout=idle_seconds)", source) + self.assertIn("signal.SIGTERM", source) + self.assertIn("fail_expired", source) + + def test_worker_schema_gate_uses_pipeline_principal(self): + source = (ROOT / "src/pipelines/worker.py").read_text(encoding="utf-8") + self.assertIn("pipeline_sessions = database.pipeline_sessions()", source) + self.assertIn("async with pipeline_sessions() as session", source) + self.assertIn("handlers = build_handlers(pipeline_sessions)", source) + + def test_backlog_retirement_is_explicit_terminal_and_preserves_rows(self): + source = (ROOT / "src/pipelines/job_admin.py").read_text(encoding="utf-8") + self.assertIn("WHERE job_run_id=ANY($1::uuid[])", source) + self.assertIn("status='cancelled'", source) + self.assertIn("started_at=COALESCE(started_at, requested_at)", source) + self.assertIn("PreWorkerDeploymentRetirement", source) + self.assertIn("INSERT INTO job_events", source) + self.assertNotIn("DELETE FROM job_runs", source) + self.assertNotIn("TRUNCATE", source) + + +if __name__ == "__main__": + unittest.main()