diff --git a/changelog.d/3010.security.md b/changelog.d/3010.security.md index d7f916ba1..feab2bbe7 100644 --- a/changelog.d/3010.security.md +++ b/changelog.d/3010.security.md @@ -1,17 +1,30 @@ -- **An LDAP search that does not end with success is an outage, never "no such account".** Each - search the engine sends refuses any result code but success, with or without entries. That - includes busy, unavailable, unwillingToPerform and adminLimitExceeded. It includes a time or size - limit, and noSuchObject on a missing search base. Before, such a search read as no entries. The - session reconciler would then have struck a present account toward revocation. A busy nested - group search read as fewer groups. The reconciler now reads these as unavailable, which never - revokes. A sign-in writes `auth.login_error` naming the code, and `check-privileges` names the - code for the bind-account read. These refusals name the operation and the code, never the - directory's diagnostic message. Other directory errors keep the text they had. +- **An LDAP search that does not end with success is never "no such account".** Each search the + engine sends refuses any result code but success, with or without entries. That includes busy, + unavailable, unwillingToPerform and adminLimitExceeded. It includes a time or size limit, and + noSuchObject on a missing search base. Before, such a search read as no entries. The session + reconciler would then have struck a present account toward revocation. A busy nested group + search read as fewer groups. A sign-in now writes `auth.login_error` naming the code, and + `check-privileges` names the code for the bind-account read. These refusals name the operation + and the code, never the directory's diagnostic message. Other directory errors keep the text + they had. + **The session reconciler tells a configuration refusal from an outage.** noSuchObject, + invalidDNSyntax and insufficientAccessRights point at the configuration: a search base that was + deleted or moved, or that the service account may not read. Such a refusal leaves that account + unjudged and never revokes it. Every pass with one logs an ERROR naming the code, audits + `auth.ad_reconcile_misconfigured`, and raises `ad_reconcile_aborted` with reason + `directory_misconfigured` under the source `directory-reconciler-misconfigured`. A pass where + every lookup is refused, faulted or fails, at least one refused and none referred, aborts as + `directory_misconfigured`. Before, a deleted search base read as "not found" for every account, + which tripped the mass-revoke breaker on a large enough estate. Nothing resolves the new alert + automatically; an operator does. Any other code is an outage, which never revokes and pages + nobody. A second-factor step-up or a passkey assertion, which probes the directory before it + grants, refuses a refused account with outcome `misconfigured`. The password step-up and the + federated step-up refuse it as `directory_unavailable`, as they do any directory error. **Not covered:** ldap3 fetches a ranged `memberOf` with follow-up searches of its own, and those are not refused. **The cost:** a code that recurs on every pass, such as a deleted search base, stops revocation - and the role re-diff for as long as it lasts. It warns once per process for each operation and - code, and latches no alert. + and the role re-diff for as long as it lasts. A configuration code alerts; any other code warns + once per process for each operation and code, and latches no alert. **The nested group search is paged**, at 500 entries per page. So an account in more groups than a directory's unpaged cap no longer fails sign-in with sizeLimitExceeded. AD's cap, MaxPageSize, is 1000 by default. A page that does not end with success refuses the whole search. So does a @@ -34,7 +47,7 @@ **A bind response carrying a referral is a referral.** ldap3 cannot decode one. The boundary refuses it as a referral, naming no host. A sign-in writes `auth.login_error`, and the session reconciler reads it as referred and raises the `ad_reconcile_aborted` alert with reason - `directory_referral`, as on any referral. That alert's text points at the two search bases, not - at `[auth].ad_bind_dn`. + `directory_referral`, as on any referral. That alert's text names the two search bases and + `[auth].ad_bind_dn`, any of which may name another domain of the forest. Before, a sign-in that met one failed with an unhandled error, and the reconciler read it as a probe fault. (vault `BACKLOG #3010`) diff --git a/changelog.d/3028.security.md b/changelog.d/3028.security.md new file mode 100644 index 000000000..4d71abbec --- /dev/null +++ b/changelog.d/3028.security.md @@ -0,0 +1,12 @@ +- **A sign-in now ends the session its presented token became, when an elevation finished first.** + A console sign-in ends the session its browser presented. An elevation in another tab that + finished during the sign-in used to leave that token naming no session, so the session lived on + under its new token. Each rotation now records which token hash replaced the old one, in its own + store transaction, in a new `session_lineage` table on SQLite, Postgres and SQL Server. The + supersession follows those links to the live session and ends it, in the same transaction as the + revoke, for `[auth].oidc_flow_ttl_seconds` plus 120 seconds after each rotation; the session reaper + deletes older links. A retired token still authenticates nothing: no session read consults the + links. `docs/SECURITY.md` states the bound that remains, and ADR 0210 records the design. + On the server backends the schema change moves the ADR 0064 schema hash: under their default + `[store].schema_management`, `external`, the engine refuses to open until `provision-schema` has + run; under `auto` the next open runs the DDL. diff --git a/changelog.d/3120.security.md b/changelog.d/3120.security.md new file mode 100644 index 000000000..8dedf6f12 --- /dev/null +++ b/changelog.d/3120.security.md @@ -0,0 +1,21 @@ +- **A lock notice the relay's full queue drops no longer holds the next one back.** The security + notifier now says whether it queued a notice, and the `auth.lock_notice` row records + `mailed: false` for one it dropped, so the next lock of the kind mails again rather than staying + quiet for 24 hours. The same holds at shutdown: a lock notice that reaches the notifier once it + has begun to close, which a cut shutdown drain allows, is refused and recorded `mailed: false` + rather than reported queued for a mail that is never sent. The relay also records such a drop, + with the new undelivered-notice reason `closing`. A send that fails later, on the + relay's own task, still leaves `mailed: true`; `docs/SECURITY.md` states that residual. + **The lock notice's throttle read has an index.** A new `ix_audit_actor` index on + `audit_log(actor, action, id)`, on SQLite, Postgres and SQL Server, lets the read find one + account's rows instead of walking every recent notice row of every account. On Postgres it keys + the first 200 characters of the actor, since a whole overlong value would fail the audit write. + A companion `ix_audit_actor_id` index on `audit_log(actor, id)`, keyed the same way, keeps the + reads filtered by actor without an action equality (`GET /audit?actor=`, its export pages, + `/me/security-events`) + from sorting all of an actor's rows to return the newest few. + On the server backends the schema change moves the ADR 0064 schema hash: under their default + `[store].schema_management`, `external`, the engine refuses to open until `provision-schema` has + run; under `auto` the next open runs the DDL. + **Declined, with residuals in `docs/SECURITY.md`:** two mails across engine shards serving their + own API ports, the timing of the content-free failure lines, and a cap on pending notice tasks. diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 900837501..634f18fd3 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -1,11 +1,12 @@ -- **The session reconciler counts a referral over the lookups it sent, and strikes an id-less row - on every pass.** A directory row with no `objectGUID` is judged without a lookup, so no referral - can answer it. The referral's `auth.ad_reconcile_referred` audit row now carries the `asked` - field in place of `probed`, and that row, the referral's latched message and its - `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still - counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or aborted on referrals or probe faults, as on any other pass. Such a pass still revokes - nothing. Only a - pass that is not aborted can, once the row's strikes reach the threshold and the row is in that - pass's probe budget, so while every pass aborts its sessions last to their own expiry. Its revocation still counts toward the mass-revoke breaker, as before. - (`vault BACKLOG #3170`) +- **The session reconciler counts a referral or a probe fault over the lookups it sent, and strikes + an id-less row on every pass.** A directory row with no `objectGUID` is judged without a lookup, + so no referral or fault can answer it. The `auth.ad_reconcile_referred` and + `auth.ad_reconcile_faulted` audit rows now carry the `asked` field in place of `probed`. Each + row, its latched message and its `ad_reconcile_aborted` alert count only the lookups the pass + sent. The breaker's alert still counts every account the pass judged, its denominator. + An id-less row is now struck on every pass, as on a pass that judged the directory's answers. + That includes a directory outage, and a pass aborted because every lookup was referred, faulted + or failed. No aborted pass revokes the row. Only a pass that is not aborted can, once the row's + strikes reach the threshold and the row is in that pass's probe budget. So while every pass + aborts, its sessions last to their own expiry. Its revocation still counts toward the + mass-revoke breaker, as before. (`vault BACKLOG #3170`) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 5f8d28bdf..d51c4bae6 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -766,11 +766,11 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `ad_session_recheck_seconds` | int | `300` | **Directory session reconciliation** ([ADR 0079](adr/0079-kerberos-idp-session-coordination.md) mechanism 2). How often to re-resolve directory principals holding **live** sessions and revoke those AD has disabled or deleted — without it, an AD disable does not take effect until the `[security].max_session_hours` cap (12 h). **`300` (five minutes) is the default** (ADR 0148 GIVEN 1 — the hardened path is the shipped path), floored at **60 s** (a pass costs one LDAP bind per signed-in directory user). `0` disables the loop and is a **loosening** once AD is on — `security_loosenings()` names it. The default is **inert without AD** (`should_reconcile()` also needs an LDAP client), so a non-AD deployment is unaffected; an **explicit** non-zero value without `ad_enabled` is still refused rather than left silently dead. | | `ad_session_recheck_strikes` | int | `2` | Consecutive passes a principal must fail to resolve before its sessions are revoked. *The search matched nothing* cannot tell *deleted* from *moved out of the search base*, so a single ambiguous result must never revoke; a set disabled bit and an unreadable `userAccountControl` strike the same way. A wave of unreadable answers is held instead of revoked, with no setting ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). Range 1–10. | | `ad_session_recheck_max_users` | int | `200` | Per-pass bind budget. Beyond this, remaining users are picked up by later passes (least-recently-probed first), so a large estate degrades to a longer effective interval instead of a bind storm. A pass revokes no more accounts than it probed. So at or below `ad_session_revoke_max`, the breaker can never trip. With AD on and a non-zero interval, the engine then logs a **WARNING** at start that names both settings. It still loads, and revocation still works. To arm the breaker, raise this budget above the cap. | -| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A bad search base / moved OU / service account that lost read rights on the entries answers "not found" for *every* user — indistinguishable from "everyone was deleted". A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). At `0` the absolute half is gone, so a pass trips once it revokes more than the fraction of the accounts it probed. At or above `ad_session_recheck_max_users`, the breaker can never trip; that row says what the engine does. | +| `ad_session_revoke_max` | int | `5` | **Mass-revoke circuit breaker**, absolute half. A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or a service account that lost read rights on the entries while it can still read the base answers "not found" for *every* user — indistinguishable from "everyone was deleted". A base that was deleted or moved, or that the service account may not read at all, usually ends the search with a result code instead: a configuration refusal, which raises its own `ad_reconcile_aborted` instance with reason `directory_misconfigured` on any pass that sees one. The refused account is neither struck nor revoked, but it still counts among the accounts the pass probed, the breaker's denominator. When such a pass aborts and revokes nothing before the breaker runs is stated once in [SECURITY.md](SECURITY.md) (BACKLOG #3010). A lost read right on `userAccountControl` alone is held instead ([ADR 0195](adr/0195-brake-the-ad-session-reconciler-on-an-undetermined-useraccountcontrol-wave.md)). At `0` the absolute half is gone, so a pass trips once it revokes more than the fraction of the accounts it probed. At or above `ad_session_recheck_max_users`, the breaker can never trip; that row says what the engine does. | | `ad_session_revoke_max_fraction` | float | `0.34` | Circuit breaker, proportional half. A pass exceeding **both** thresholds aborts, revokes nothing, logs at ERROR and writes an `auth.ad_reconcile_aborted` audit row. Requiring both means it fires only on a change simultaneously *large* and *broad* — the signature of a misconfiguration, not of offboarding (3-of-3 or 50-of-300 still applies). Range >0.0–1.0, but with AD on and a non-zero interval the engine refuses `1.0` at load. At `1.0` this half is never exceeded, for the reason the `ad_session_recheck_max_users` row gives, so the whole breaker would never trip. | | `kerberos_enabled` | bool | `false` | Windows SSO (experimental, off by default; needs `ad_enabled`) | | `kerberos_spn` | str | — | service principal, exactly `SERVICE/host`, e.g. `HTTP/host.example.com`. The engine splits it at the `/` into pyspnego's `service` and `hostname`. A malformed value is **refused at load**, including at least one with no `/`, more than one `/`, an empty half, whitespace or control characters, or a realm suffix (`HTTP/host@REALM`); the host's own domain supplies the realm. | -| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**. An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | +| `oidc_enabled` | bool | `false` | Federated SSO — OIDC auth-code + PKCE relying party ([ADR 0142](adr/0142-federated-sso-oidc-authorization-code-pkce-relying-party-hybrid-ad-backed.md)). A third login for an identity that **already exists in on-prem AD** (needs `ad_enabled`; roles come from LDAP, not the token). Off = byte-identical **for an install with no bound account**, with at least these exceptions: `oidc_flow_ttl_seconds` still sets the session lineage window (its row says how), and the single-key checks on the `oidc_*` keys, such as the `oidc_flow_ttl_seconds` range, still run at load, as does the check that `oidc_callback_min_elapsed_seconds` is shorter than the TTL. An account an administrator has bound to a federated identity signs in through the IdP only: Windows SSO refuses it whatever this key says, so turning this off leaves a bound account with no sign-in until it is unbound ([SECURITY.md](SECURITY.md#federated-sign-in-oidc-browser-only--adr-0142), vault BACKLOG #2609). Needs `[security].web_console_public_address` (the redirect origin). | | `oidc_issuer` | str | — | https; exact-matched against the id_token `iss`. At most 256 UTF-16 code units, which is 256 characters for an ASCII URL, **refused at load** beyond that: an administrator's bind stores it, and the narrowest issuer column (SQL Server `NVARCHAR(256)`) holds no more (BACKLOG #2331) | | `oidc_client_id` | str | — | also the required `aud`/`azp` | | `oidc_client_secret` | str | — | confidential-client secret — supply via env (`MEFOR_AUTH_OIDC_CLIENT_SECRET`) or `oidc_client_secret_ref`. A value in the file is still accepted, with a WARNING at load; keep it out of the file | @@ -798,7 +798,7 @@ document ([SECURITY-DOCS-POLICY.md](SECURITY-DOCS-POLICY.md)). | `oidc_mfa_amr_values` / `oidc_required_acr_values` | list[str] | `["mfa"]` / `[]` | either family satisfies the gate; both empty with the gate on is refused. Each value is stripped at load and a blank one dropped, in a TOML list as in an env string, so a list of blanks counts as empty ([BACKLOG #2325](BACKLOG.md)) | | `oidc_acr_values` / `oidc_prompt` | str | — | requested authorize params. `oidc_acr_values` is a request only: the gate checks the returned `acr` against `oidc_required_acr_values` alone, and only while `oidc_require_mfa_claim` is on. So while `oidc_enabled` is on, a non-blank `oidc_acr_values` is **refused at load** if `oidc_required_acr_values` names no non-blank value (BACKLOG #2032). It is also refused while `oidc_require_mfa_claim` is off, whatever the required list holds (BACKLOG #2325). A whitespace-only `oidc_acr_values` or `oidc_prompt` loads as no request and is not sent (BACKLOG #3221 for `oidc_prompt`). A token whose `amr` matches `oidc_mfa_amr_values` still passes whatever its `acr`; to rely on `acr` alone, also empty `oidc_mfa_amr_values` | | `oidc_jwks_ttl_seconds` / `oidc_jwks_min_refetch_seconds` | int | `3600` / `300` | the JWKS cache TTL + the amplification (min-refetch) bound | -| `oidc_flow_ttl_seconds` / `oidc_flow_cache_max` | int | `300` / `512` | pending-flow TTL + the **reject-when-full** bound | +| `oidc_flow_ttl_seconds` / `oidc_flow_cache_max` | int | `300` / `512` | pending-flow TTL + the **reject-when-full** bound. `oidc_flow_ttl_seconds` also sets the session lineage window, `oidc_flow_ttl_seconds` + 120 s, and it does so whether or not OIDC is on: for that long after an elevation re-keys a session, a sign-in that presents the retired token can still end the session it became, and the session reaper purges older lineage links. A longer TTL lengthens that window; a retired token never authenticates. When a sign-in still misses the session is stated once in [SECURITY.md](SECURITY.md#session-inventory--targeted-revocation-wp-10). Why the window is that long is in [ADR 0210](adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md) and in `AuthService.supersession_lineage_seconds` in `messagefoundry/auth/service.py` (vault BACKLOG #3028) | | `oidc_callback_min_elapsed_seconds` | float | 1.0 | the least time between a federated start and its callback ([BACKLOG #2301](BACKLOG.md), ASVS 2.4.2). A step-up callback that returns sooner is refused. A sign-in callback is refused the same way, but only when the verified `auth_time` shows the person signed in at the IdP during this flow: an IdP holding a live single sign-on session answers with no human step, and there is nothing to floor. The refusal is the leg's ordinary one (`federated sign-in failed`, or the generic step-up refusal), audited with `reason=too_early`. The default is a **provisional** human-timing floor, derived with `mfa_verify_min_elapsed_seconds`. `0` turns it off; it must be shorter than `oidc_flow_ttl_seconds` | | `oidc_callback_floor_exempt_amr` | list[str] | `[]` | the `amr` values that mark a re-authentication with no human step, such as integrated Windows sign-in or a client certificate ([BACKLOG #2388](BACKLOG.md)). A sign-in or step-up callback whose signature-verified `amr` names one skips `oidc_callback_min_elapsed_seconds`. Every other callback keeps the floor. Values are stripped and a blank one dropped, as for `oidc_mfa_amr_values`. A value `oidc_mfa_amr_values` also accepts is **refused at load** while `oidc_require_mfa_claim` is on. The matched values are recorded on audit rows as `callback_floor_exempt_amr`; [SECURITY.md](SECURITY.md), *What these floors do not do*, says which rows. With the list set, a too-early step-up's code is redeemed before it is refused, and a failed exchange is refused with its own reason. Any value is a **named loosening**: it proves a device answered, not that a person acted | | `oidc_session_max_hours` | int | — | caps the federated session below `id_token.exp` if a tighter bound is wanted (ADR 0079 mechanism 1) | @@ -844,9 +844,10 @@ three on its own, until it restarts again. - An LDAP referral opens its own `ad_reconcile_aborted` instance, with reason `directory_referral` and source `directory-reconciler-referral` (BACKLOG #2538). It resolves once every account signed in at the last referral has since been read in full, present and enabled, on a pass with no - referral. A pass with any referral or probe fault resolves none of these alerts. Neither is + referral. A pass with any referral, probe fault or configuration refusal (BACKLOG #3010) + resolves none of these alerts. None is carried to later passes for the breaker's and the hold's alerts: a later pass whose probe sample - leaves out the referred or faulting account reads its older answer, and can resolve one of + leaves out the referred, faulting or refused account reads its older answer, and can resolve one of those two (BACKLOG #3100, left open). - So an account that never reads in full keeps the referral's alert open while it is signed in. A held account, a disabled one awaiting its strikes, and one that never answers all do this. @@ -863,6 +864,14 @@ source `directory-reconciler-probe-fault` (BACKLOG #3100). The reconciler raises which a probe fails to read an entry. Nothing resolves it automatically. Resolve it by hand once the WARNING lines that name each faulting account stop. +A configuration refusal opens its own `ad_reconcile_aborted` instance as well, with reason +`directory_misconfigured` and source `directory-reconciler-misconfigured` (BACKLOG #3010). The +reconciler raises it on every pass in which the directory refuses a search with noSuchObject, +invalidDNSyntax or insufficientAccessRights, as a deleted or moved search base does. Before #3010 +such a base read as "not found" for every account. Nothing resolves it automatically either. A +search base changes only on a restart, so resolve it by hand once the base is fixed and the ERROR +line naming the code stops. + A paging refusal opens one `ad_reconcile_aborted` instance per account, under the source `directory-reconciler-paging:` (vault BACKLOG #3696). Nothing resolves it automatically. Resolve it by hand once that account no longer hits the bound, for example once its sign-in diff --git a/docs/CONNECTIONS.md b/docs/CONNECTIONS.md index 581f216e8..415c4b68a 100644 --- a/docs/CONNECTIONS.md +++ b/docs/CONNECTIONS.md @@ -4009,7 +4009,7 @@ reading this page already applies to a file the scan never opened. | Store — SQL Server (`[store].backend = sqlserver`) | three distinct bounds: `[store].connect_timeout` 15 s (**login**, passed as pyodbc `timeout=`), `[store].command_timeout` 30 s (**statement**, applied per acquire as the pyodbc connection attribute) and `[store].acquire_timeout` 30 s (the **pool borrow**); `[store].warm_pool_timeout` 15 s bounds the warm-up | every acquire releases back to the pool; the pool is closed on shutdown; a borrow the pool satisfies after the borrower gave up is released back rather than stranded | a driver error — or a `StoreAcquireTimeout` — propagates to the stage worker and the row stays claimable | stage handoffs re-run idempotently; `reset_stale_inflight` recovers on restart | | Store — Postgres (`[store].backend = postgres`) | `[store].connect_timeout` 15 s (`create_pool(timeout=…)`), `[store].command_timeout` 30 s as `asyncpg`'s per-statement bound, and `[store].acquire_timeout` 30 s on the message-pipeline pool borrows (see the coverage note above) | as SQL Server | as SQL Server | as SQL Server | | Active Directory — login binds (`[auth].ad_server`) | `[auth].ad_connect_timeout` 10 s on the LDAP TCP connect and `[auth].ad_receive_timeout` 10 s on every LDAP response read — threaded into **every** `ldap3` `Server`/`Connection` construction. The nested group search is paged, and **all its pages share one `[auth].ad_group_search_deadline`** 120 s: each socket read of a page after the first waits at most the shorter of `ad_receive_timeout` and what was left when the page was asked for, no page is asked for once it is spent, and the abandon of a refused search gets only what is left (vault BACKLOG #3696). **It is not a hard wall-clock bound**: the cap is set once per page and applies to each read, so a page that arrives over several reads, each inside that cap, can hold the search up to the number of reads times what was left. The page size is the directory's choice, since it may ignore the 500 the engine asks for | the service-account connection is context-managed; the user bind is unbound in a `finally`, so a **rejected** password releases it too (the common adversarial case) | fails closed with `LdapError`; the login is rejected and audited, and so is a group search past its deadline or past 10,000 groups, whatever page size the directory returns (one WARNING per process for each) | **single-shot** — two binds, no retry loop; the group search's pages are one search, not retries | -| Active Directory — session reconciler (`ad_session_recheck_seconds`) | the same `ad_connect_timeout` / `ad_receive_timeout` / `ad_group_search_deadline` | as the login path — context-managed connections | a pass that fails is retried on the next interval; strike state is process-local. A group search refused for its deadline, its 10,000-group cap or its page backstop reads that account as unavailable on every pass. A disable or deletion still revokes it, because the user search runs first, but its roles and channel scope are not re-diffed while that lasts. So it pages, as `ad_reconcile_aborted` under the source `directory-reconciler-paging:`, and audits `auth.ad_reconcile_paging_refused`, once as the account enters that state, never on every pass. It pages again only after a pass judges the account, which an outage, a referral or a probe fault does not, or after a restart (vault BACKLOG #3696). A plain outage still pages nobody | **single-shot per pass**; `ad_session_recheck_strikes` (2) required before a revoke | +| Active Directory — session reconciler (`ad_session_recheck_seconds`) | the same `ad_connect_timeout` / `ad_receive_timeout` / `ad_group_search_deadline` | as the login path — context-managed connections | a pass that fails is retried on the next interval; strike state is process-local. A group search refused for its deadline, its 10,000-group cap or its page backstop reads that account as unavailable on every pass. A disable or deletion still revokes it, because the user search runs first, but its roles and channel scope are not re-diffed while that lasts. So it pages, as `ad_reconcile_aborted` under the source `directory-reconciler-paging:`, and audits `auth.ad_reconcile_paging_refused`, once as the account enters that state, never on every pass. It pages again only after a pass judges the account, which an outage, a referral, a configuration refusal or a probe fault does not, or after a restart (vault BACKLOG #3696). A plain outage still pages nobody | **single-shot per pass**; `ad_session_recheck_strikes` (2) required before a revoke | | Kerberos / SPNEGO SSO (`kerberos_spn`) | **none engine-owned** — the OS provider owns any KDC timeout | the SPNEGO context is per-request | a failed step raises `LdapError` and audits a login reject | **single-shot**, single-leg — no NTLM fallback, no multi-leg handshake | | OIDC IdP — token endpoint (`oidc_token_endpoint`) | 10 s — `auth/oidc/flow.py`'s **own** `exchange_code(timeout=…)` default (`AuthService._oidc_exchange` passes none). The JWKS leg's 10 s is a **separate** literal, `oidc_http.DEFAULT_IDP_TIMEOUT_SECONDS`; the two coincide but are independent | the response is context-managed; the body is size-capped | a non-200 or oversize body fails the login closed | **single-shot** — one POST per login | | OIDC IdP — JWKS fetch (`oidc_jwks_uri`) | 10 s — `oidc_http.DEFAULT_IDP_TIMEOUT_SECONDS`, the constant `jwks_fetcher` is the only consumer of | the response is context-managed; the size cap is enforced on the socket read | a fetch failure fails verification closed | **single-shot**, further damped by the refetch floor | diff --git a/docs/PHI.md b/docs/PHI.md index 0ae730e32..4155b5bf8 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -200,6 +200,7 @@ destruction) are documented in [§3](#3-encryption-at-rest) under the matching h | `state.namespace` / `state.key` | all three | **Possibly** — a Handler that keys correlation state on a raw MRN stores that identifier here in the clear | **No** — plaintext by construction: the pair is the composite primary key **and** the AAD input for `state.value`, so it cannot be ciphered without losing the lookup | **PL-4** | Authors must key state on a **surrogate, never a raw identifier**. Covered only by the whole-DB / volume layer. Rides `[retention].state_max_age_days` with its value | ``rides `[retention].state_max_age_days` `` | | `reference.name` / `reference.version` / `reference.key` | all three | **Possibly** — §2's `reference.value` row notes a snapshot row may be patient-keyed; the key column is where that identifier would sit | **No** — plaintext by construction, same reason as `state` | **PL-4** | Same rule: key reference sets on a surrogate. The key columns ride the same delete: `purge_reference_snapshots` removes whole rows of an **undeclared** set, so these go with the `value` they key. A **declared** set is never touched — same orphan-only limit as `reference.value` above | ``orphan-only `[retention].reference_snapshot_days` `` | | `sessions.token_hash` / `client`, `resend_log`, `processed_files`, `pending_approvals.params`, `webauthn_credentials.public_key` | all three | **No** | No (deliberately not ciphered) | **PL-4** | Session tokens are stored as SHA-256 only; `processed_files` holds a hashed derived file key, never a path; approval params carry connection names / channel ids / a config dir by construction; COSE public keys (ADR 0068) are **verification material, not secrets** and are explicitly excluded from the cipher and from rekey | `n/a — not PHI` | +| `session_lineage` (`retired_hash` / `successor_hash` / `retired_at`, vault BACKLOG #3028, ADR 0210) | all three | **No** | No (deliberately not ciphered) | **PL-4** | Two SHA-256 session token hashes and a time, the same class as `sessions.token_hash`: which hash an elevation retired and which replaced it. Read only by the sign-in supersession, never by a session read, so a retired hash authenticates nothing; not sealed, because a forged link can only direct a revoke. The session reaper deletes links past the window the supersession follows them for (section 8) | `n/a — not PHI` | | `known_login_addresses` (`user_id` / `address` / `first_seen` / `last_seen`, vault BACKLOG #2145) | all three | **No** — a network address tied to an account; identifies a *host*, not a patient | No (metadata, deliberately not ciphered) | **PL-4** | The first-seen sign-in address baseline (BACKLOG #288): one row per host an account finished a sign-in from, or, in the cases [SECURITY.md](SECURITY.md) Table A lists, passed a step-up or confirmed a factor enrolment from, keyed on the account id. **Personal data, not PHI** — the same class and the same decision as `sessions.client` and `audit_log.client`. Read only by the sign-in signal; **no API surface**. Each write to the account's record, even one that re-records a known address, is followed by a prune of its rows past the signal's 90-day lookback. An event that writes nothing prunes nothing, such as a sign-in that still owes a factor. A failed prune deletes nothing either. So a dormant account keeps its stale rows until its next write. An administrator's password or MFA reset deletes the account's rows and leaves one marker row holding no address (vault BACKLOG #2946). `delete_user` removes every row with the account, and the account's foreign key cascades to any row a racing write leaves | `n/a — not PHI` | | `secret_rotation_meta` (`secret_key` / `fingerprint` / `tracked_since` / `last_rotated`) | **All three backends** (#1186 — SQLite `MessageStore`, `SqlServerStore` and `PostgresStore` each create the table at open and implement the `SecretRotationMetaStore` protocol) | **No** — non-secret rotation state: a **keyed MAC** (DEK-derived, one-way — never the secret value) + ISO dates only | No (deliberately not ciphered — it is neither PHI nor a secret; the MAC is one-way and un-guessable without the DEK) | **PL-4** | ASVS 13.3.4 rotation watcher (BACKLOG #282). `fingerprint` is a keyed MAC so obtaining the rows leaks no secret. Written on a keyed store; absent on a keyless one | `n/a — not PHI` | | SQLite DB file + `-wal` / `-shm` / temp files, and every index | SQLite | **Yes** (mirror the above) | **No** — the app cipher cannot reach them | **PL-5** | WAL/shm hold recently-written PHI outside any app-level encryption. Cover: SQLCipher (whole-DB) and/or FDE on the engine host's data volume. A purge zeroes what it frees, and the WAL truncate is tried hourly, both by default; what they miss is in [§8](#8-retention--purge) (vault BACKLOG #2619) | `n/a — not PHI` | @@ -996,7 +997,9 @@ on the store-file ACL plus the volume/whole-DB layer for the rest. - `audit_log`, `delivered_keys`, `resend_log` — **keep-forever by design**; `[retention].audit_days` is **reserved and not enforced**. - `sessions` — expired rows are deleted by `purge_expired_sessions`, driven from the auth layer (idle - 30 min / absolute 12 h), not by the RetentionRunner. + 30 min / absolute 12 h), not by the RetentionRunner. The same pass deletes `session_lineage` links + (two token hashes and a time, vault BACKLOG #3028) once they are past the window the sign-in + supersession follows them for. - `processed_files` — an age/count prune driven from the wiring runner, also outside the RetentionRunner. - `state.namespace` / `state.key` — removed with their value on `[retention].state_max_age_days`. - `reference.*` — **no purge path**; replaced only by the next sync's build-new-then-flip. @@ -1821,7 +1824,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP and HTTP listeners' `tls_handshake_failed` (vault BACKLOG #2613 and #3302 -- an inbound TLS handshake failed, on any event loop; the first at once, then one per listener per minute carrying the count since the last. The reason is a count, an exception class name and OpenSSL's reason mnemonic, never the peer's bytes or certificate. The event names a peer only where the MLLP listener runs its own handshakes, which is the standard asyncio loop; where the loop runs them, which is uvloop and every HTTP listener, the peer is not known, and a handshake that only runs past its bound is not counted), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100, or an account's group search was refused for the engine's own deadline or cap, so a role or scope change in the directory does not revoke that account's sessions, though a disable or deletion still does, vault BACKLOG #3696; for the breaker, a referral and a probe fault the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker judged; for a probe fault the same total of candidates the pass took up, asked or not, faulted ones included; or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username; a paging refusal is instead keyed on that operator account's username, as `directory-reconciler-paging:`, so each account is its own instance, raised as the account enters that state and again after a restart while it stands, and the payload is that key, the closed-set reason slug `group_search_deadline` or `group_search_cap`, the same count as a probe fault's and a fixed sentence naming the cause, never anything else about the account), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), `sandbox_worker_refused` (a `[sandbox].mode = "subprocess"` worker was refused because the graph it loaded differs from the engine's, most often a config file edited on disk with no reload -- vault BACKLOG #2724; keyed on the inbound's name, raised once per refusal, and the payload is that name and a fixed sentence naming the cause and the remedy, never a message body or the refusal text, which names config names and digests and is in the application log), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` under the same key, so for the same file and the same engine identity, at least once the reload's record no longer holds the refusal, once no hop holds any CRL, and when the reload stops -- vault BACKLOG #2904; it is also raised once per held file outside a CA bundle, at the first pass that judges it with no refusal standing, whether or not an alert is open, so it can resolve nothing, and it then clears one an earlier run under that identity left open -- vault BACKLOG #3543. An alert for a file no hop holds now stays open, and so does one an unpinned cluster node raised, because that node's identity changes on every start; [CONFIGURATION.md](CONFIGURATION.md), *The alert resolves itself*, is the source of record), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_held` and `ad_reconcile_aborted`, the latter only under the breaker's or the referral's source: the probe fault's and a paging refusal's instances have no inverse; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100, or the directory refused one or more probes with a result code that points at the configuration, such as a deleted search base, and those accounts were left unjudged the same way, vault BACKLOG #3010, or an account's group search was refused for the engine's own deadline or cap, so a role or scope change in the directory does not revoke that account's sessions, though a disable or deletion still does, vault BACKLOG #3696; for the breaker, a referral, a probe fault and a configuration refusal the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral, `directory-reconciler-probe-fault` for a probe fault and `directory-reconciler-misconfigured` for a configuration refusal so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral`, `probe_fault` or `directory_misconfigured`, a count (the accounts the breaker judged, or for each of the other three the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username; a paging refusal is instead keyed on that operator account's username, as `directory-reconciler-paging:`, so each account is its own instance, raised as the account enters that state and again after a restart while it stands, and the payload is that key, the closed-set reason slug `group_search_deadline` or `group_search_cap`, the same count as the breaker's and a fixed sentence naming the cause, never anything else about the account), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), `sandbox_worker_refused` (a `[sandbox].mode = "subprocess"` worker was refused because the graph it loaded differs from the engine's, most often a config file edited on disk with no reload -- vault BACKLOG #2724; keyed on the inbound's name, raised once per refusal, and the payload is that name and a fixed sentence naming the cause and the remedy, never a message body or the refusal text, which names config names and digests and is in the application log), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` under the same key, so for the same file and the same engine identity, at least once the reload's record no longer holds the refusal, once no hop holds any CRL, and when the reload stops -- vault BACKLOG #2904; it is also raised once per held file outside a CA bundle, at the first pass that judges it with no refusal standing, whether or not an alert is open, so it can resolve nothing, and it then clears one an earlier run under that identity left open -- vault BACKLOG #3543. An alert for a file no hop holds now stays open, and so does one an unpinned cluster node raised, because that node's identity changes on every start; [CONFIGURATION.md](CONFIGURATION.md), *The alert resolves itself*, is the source of record), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_held` and `ad_reconcile_aborted`, the latter only under the breaker's or the referral's source: the probe fault's, the configuration refusal's and a paging refusal's instances have no inverse; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 9e0773243..fc9cd29f6 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4584,10 +4584,11 @@ writes no roles, and it strikes and revokes as an absent account does, with the `directory_object_id_missing`. It is not a `userAccountControl` answer, so it never moves the ADR 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, -and one that aborts because its lookups were only referred, faulted or failed, still strikes the -row, because its answer never came from the directory (BACKLOG #3170). No such pass revokes it: only a pass that is not aborted can, once -the row's strikes reach the threshold and the row is in that pass's probe budget. A referral's `auth.ad_reconcile_referred` row and alert count only -the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a +and one that aborts because its lookups were only referred, refused for the configuration, faulted +or failed, still strikes the row, because its answer never came from the directory (BACKLOG #3170). +No such pass revokes it: only a pass that is not aborted can, once the row's strikes reach the +threshold and the row is in that pass's probe budget. The audit rows and alerts of a referral, a +configuration refusal and a probe fault count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less row, made before that refusal, is **never probed by name** (BACKLOG #2027). The pass skips the row and @@ -4599,7 +4600,8 @@ disable or demotion no longer ends that row's sessions within one interval, only Removing the binding (`DELETE /users/{user_id}/federated-identity`) returns the row to the pass. The pass has these safety properties, because the lookup still returns one indistinguishable "not -found" for *deleted*, *moved out of the search base* and *the search base was never right*: +found" for *deleted*, *moved out of the search base* and *a search base that exists but was never +the right one*: - **Fail-OPEN.** An unreachable domain controller revokes **nothing**, and an account the pass asked about accrues no strike. An id-less row, never asked, is still struck (BACKLOG #3170). @@ -4625,8 +4627,12 @@ found" for *deleted*, *moved out of the search base* and *the search base was ne and no floor. **The cost:** two genuinely disabled accounts whose attribute the bind account cannot read keep their sessions to the absolute cap. -- **A mass-revoke circuit breaker.** A misconfigured `ad_user_search_base`, an OU reorganisation, or a - service account that lost read rights answers "not found" for *every* user. A pass whose revocation +- **A mass-revoke circuit breaker.** An `ad_user_search_base` that names the wrong OU, an OU + reorganisation that moves the users out of it, or a service account that lost read rights on the + user entries while it can still read the base, answers "not found" for *every* user. A base that + was deleted or moved, or that the service account may not read at all, usually does not: the + search ends with a result code, which is the configuration refusal described below (BACKLOG + #3010). Which of the two a directory answers is not measured against a domain controller. A pass whose revocation set exceeds **both** `ad_session_revoke_max` (5) **and** `ad_session_revoke_max_fraction` (0.34) of the probed population **aborts**: nothing is revoked, nothing is written, the engine logs at ERROR and audits `auth.ad_reconcile_aborted`, raises the `ad_reconcile_aborted` alert, and the condition @@ -4650,16 +4656,14 @@ revoked. A pass with any referral audits `auth.ad_reconcile_referred` and raises `ad_reconcile_aborted` alert with reason `directory_referral`, under its own source `directory-reconciler-referral`. So it is its own instance, apart from the breaker's, and acknowledging or suspending one does not silence the other. An `[alerts]` rule keyed on the event -type alone matches the breaker, the referral and the probe fault, and so does a source glob such -as `directory-reconciler*`, which also matches the hold's `ad_reconcile_held`. To mute only -the referral, a rule names its source exactly. The alert's text names `ad_user_search_base` and -`ad_group_search_base`. A bind response that carries a referral, the service account's included, -is a referral too (BACKLOG #3010). ldap3 cannot decode its hosts, so the ERROR line names none. The -alert's text still points at the two search bases and not at `ad_bind_dn`, the DN that bind used, -so for this case it names the wrong setting. A service-account bind answered with the referral -code and no referral field is refused as a referral as well (BACKLOG #3017), with the same wrong -setting named. When it resolves itself -is stated once, in +type alone matches the breaker, the referral, the probe fault, the configuration refusal and each +paging refusal, and so does a source glob such as `directory-reconciler*`, which also matches the hold's +`ad_reconcile_held`. To mute only the referral, a rule names its source exactly. The alert's text +names `ad_user_search_base`, `ad_group_search_base` and `ad_bind_dn`. A bind response that carries a +referral, the service account's included, is a referral too (BACKLOG #3010). ldap3 cannot decode +its hosts, so the ERROR line names none. A service-account bind answered with the referral code +and no referral field is refused as a referral as well (BACKLOG #3017). When it resolves itself is +stated once, in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-alerts-resolve-themselves). Referred accounts stay in the mass-revoke breaker's count, as unreachable ones do. Left out, a referring group base would trip the breaker on any handful of genuine disables. A base in another domain of the forest @@ -4682,8 +4686,9 @@ directory outage (BACKLOG #3100). Each such probe is logged at WARNING by name, estate is still reconciled. A pass with any fault logs one ERROR naming no account and audits `auth.ad_reconcile_faulted`. It also raises the `ad_reconcile_aborted` alert with reason `probe_fault`, under its own source `directory-reconciler-probe-fault`. If every probe faults or -fails, the pass judges no account it asked about and revokes nothing, like an outage, but it still -pages, and it still strikes an id-less row (BACKLOG #3170). A +fails, at least one a fault, the pass judges no account it asked about and revokes nothing, like an +outage, but it still pages, and it still strikes an id-less row (BACKLOG #3170). A pass where every +probe fails and none faults is a plain outage, and pages nobody. A connection, timeout or name-lookup error that escapes the directory layer is still read as unreachable. So is at least an operating-system error saying the network or host is unreachable, down or reset. Another network-class error can still read as a probe fault and page. @@ -4693,6 +4698,26 @@ visible and limits nothing. A pass with a fault marks no alert clear. When it re [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-alerts-resolve-themselves). Step-up refuses the account with outcome `probe_fault`. +A search the directory refuses with noSuchObject, invalidDNSyntax or insufficientAccessRights is a +configuration refusal (BACKLOG #3010). A search base that was deleted or moved, or that the service +account may not read, answers so for every account on every pass. Before #3010 the engine read that +as "not found", which struck every account and tripped the breaker on a large enough estate. Nor is +it read as an outage, which would page nobody. The refused account is left unjudged: no strike, no +revocation. A pass with any such refusal logs one ERROR naming the code and no account, audits +`auth.ad_reconcile_misconfigured`, and raises the `ad_reconcile_aborted` alert with reason +`directory_misconfigured`, under its own source `directory-reconciler-misconfigured`. If every +probe is refused, faulted or fails, at least one refused and none referred, the pass aborts as +`directory_misconfigured`: it judges no account it asked about and revokes nothing, but it pages. +Like the fault's, that alert has no inverse; an operator resolves it, and a pass with such a +refusal marks no alert clear. A service-account bind the directory refuses with one of those codes +takes the same path, as a malformed `ad_bind_dn` does. **The cost:** while the +refusal lasts, the refused accounts' sessions are never revoked by the reconciler, so they run to +the absolute cap. Other result codes that are not an answer, such as busy or unavailable, are an +outage. A second-factor step-up and a passkey assertion, which probe the directory before they +grant, refuse the account with outcome `misconfigured`. At least two other step-up legs refuse it +as `directory_unavailable`, the reason each gives for any directory error: the password step-up, a +live re-bind, and the federated step-up's directory lookup. + A group search the engine refuses for its own bound, past `[auth].ad_group_search_deadline` or past its cap on groups or pages, reads that account as unreachable (vault BACKLOG #3696). A disable or deletion still revokes it, because the user search runs first, but a role or channel-scope @@ -4885,14 +4910,43 @@ engine client (`EngineClient.login`) both name the token they replace in `supers client names only a token the engine issued to it, never one adopted with `set_token`. Every elevation gives a session a new token: completing MFA, a step-up, a passkey ceremony or a -password re-check. The supersession finds and ends the presented session in one atomic store -operation, so an elevation cannot land between the two (BACKLOG #2146). If the supersession runs -first, a later elevation on that session fails and asks the user to sign in again. **One case stays -open.** An elevation in another tab can finish while the sign-in is still in progress, before the -supersession runs. That window covers the credential check, and on federated sign-in the round trip -to the identity provider. The presented token then names no session, so nothing is ended. The -session lives on under its new token until it expires, because the engine does not record which -token replaced the old one. +password re-check. The supersession finds and ends the presented session in one write on its row, +so an elevation cannot land between the two (BACKLOG #2146). If the supersession runs first, a later +elevation on that session fails and asks the user to sign in again. + +**An elevation that finished first is followed (vault BACKLOG #3028, +[ADR 0210](adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md)).** An +elevation in another tab can finish while the sign-in is still in progress, before the supersession +runs. On federated sign-in that window includes the round trip to the identity provider, because +the presented token is read on the start leg. The presented token then names no session. So each +rotation records, in its own store transaction, which token hash replaced the old one, and the +supersession follows those links, up to 32 elevations in a row, and ends the session at the live +end, in the same transaction as the revoke. The audit row names that session and, in a `presented` +field, the hash the browser presented. + +The link is followed for a bounded time: `[auth].oidc_flow_ttl_seconds` plus 120 seconds, which is +420 seconds at the default, and the session reaper deletes older links. **At least these cases stay +open**, and in each the session lives on under the token the elevation handed out, until it expires, +as before this change: + +- an elevation that a second holder of the same token completed longer ago than the bound; +- a sign-in that runs past the bound, which a slow identity provider or directory can cause, since + their timeouts bound each call and not the whole sign-in; +- more than 32 elevations on one session inside the bound; +- a wall clock that stepped between the elevation and the sign-in, which moves the bound by the + size of the step, in either direction. + +**A retired token authenticates nothing.** No session read, the validator included, consults the +links; the supersession is their one reader. For the length of the bound, a retired token has one +power: presented at a sign-in that succeeds, it ends the session it became. That power is new, and +it can only end a session, never open or extend one. Whoever held that token before the elevation +could already end the session with `POST /auth/logout`; the link lets them do it for the length of +the bound after the elevation too, through a sign-in whose first credential passes; with MFA +required, the supersession runs when the password-only session is minted, before the second factor, +so a password alone is enough. A wall +clock that stepped forward at the elevation, or back before the sign-in, lengthens that time by the +step; the reaper's clock-step guard covers its own purge, not this. The links are not under the session integrity seal (BACKLOG #2592): a forged link can only direct a +revoke, and a revoke needs no key on any backend. A session's `id` is its token hash, and that hash changes whenever the session completes MFA or a step-up. So an id shown on a sessions page can go stale. The console's revoke says "Nothing was @@ -5444,7 +5498,16 @@ the lock and its cycle count, and writes its own `auth.lock_notice` row, which i reads. With a relay wired, that read and the mail run in a background task (BACKLOG #2216). The refusal writes `auth.account_locked` first and does not wait for the task. So a large audit log would not push a refused sign-in past its padded slot. The task is queued per account and lock kind within -one API process. Two locks of one kind landing together there would send one mail. A sign-in lock +one API process. Two locks of one kind landing together there would send one mail. **One throttle +read runs at a time in each API process, across every account**, so locks on many accounts at once +cannot fill the store's read pool that sign-ins wait on; each mail then waits behind the reads +queued before it. Every backend carries an `(actor, action, id)` index for that read, keyed on a +prefix of the actor on Postgres (vault BACKLOG #3120). A companion `(actor, id)` index, keyed the +same way, serves the audit reads filtered by actor without an action equality (`GET +/audit?actor=`, its export pages, `/me/security-events`); without it SQLite sought the actor on the +first index and then sorted all of its rows. That each read seeks its index is measured on SQLite only; on Postgres and SQL Server +the planner's choice is not tested. The residuals are listed below, under *Declined, each with its +residual*. A sign-in lock notice on a TOTP-enrolled local account tells the owner to sign in with the password and the code together; a second-step notice says which factor was right and, **if the attempts were not the owner's**, to get a password reset or `admin-unlock` and replace that factor. @@ -5823,7 +5886,8 @@ and header are that route's. An administrator can still read each case in the audit trail, with the gaps listed below. No relay (`mailed: false`, `reason: no_notifier`) and no address (`mailed: false`) are on the `auth.lock_notice` row. A full queue and a failed send are on an `auth.lock_notice_undelivered` - row (BACKLOG #2383). A relay that is down still shows, on every other notice kind. An instance with no relay at all is reported at + row (BACKLOG #2383), and a full queue also reads `mailed: false` on the `auth.lock_notice` row + (vault BACKLOG #3120). So does a notice the relay refused because it had begun to close. A relay that is down still shows, on every other notice kind. An instance with no relay at all is reported at startup by the serve gate, except under `[security].enforcement = "warn"` with `[alerts].security_notifications_required = false`. Every other notice kind keeps its per-event line (BACKLOG #1139); none of them fires on a refused sign-in. The list is @@ -5873,30 +5937,67 @@ a failed TLS key-exchange pin is logged once per process. **The cost of the tee stays broken, or recovers and breaks again, is reported only by that first line. **A notice the relay loses is audited (BACKLOG #2383).** The `auth.lock_notice` row is written -when the notice is handed to the relay, as `mailed: true`. When the relay's queue is full, or the -send fails, the engine writes one more row, with the notice's recipient as the actor. A lock +when the notice is handed to the relay. It reads `mailed: true` only when the relay queued the +notice: one its full queue dropped, or one it refused because it had begun to close, reads +`mailed: false`, so the next lock of the kind mails again (vault BACKLOG #3120). When the relay's queue is full, the relay refuses the notice because it has +begun to close, or the send fails, the engine writes one more row, with the notice's recipient as +the actor. A lock notice writes `auth.lock_notice_undelivered`, hidden as above. Every other notice kind writes `auth.security_notice_undelivered`, which every `audit:read` reader sees. The detail names the -notice kind and the `reason`, `queue_full` or `send_failed`. An issuer's reminder also names the +notice kind and the `reason`, `queue_full`, `closing` or `send_failed`. An issuer's reminder also names the holder. The row never carries the address, the rest of the notice's detail, or the error. The source of record is `SecurityEventNotifier._record_undelivered` in `messagefoundry/pipeline/security_notify.py`. **Still open, at least:** -- At shutdown the notifier drains its queue after the store has closed, so a send that fails then - is not recorded. +- At shutdown the notifier closes while the store is still open (BACKLOG #3191), so a send that + fails in its final drain is recorded. That close is one bounded step of the shutdown. When the + step is cut, a notice still queued is neither sent nor recorded, and a send already under way + may still go out or fail unrecorded. A lock notice among them keeps its `auth.lock_notice` row + reading `mailed: true`, so the throttle holds back that account's next lock notice of the kind + for the rest of its 24 hours. +- A notice that reaches the notifier once it has begun to close is refused and recorded with + `reason: closing`. The close waits only for records started before it finishes. One started + later can meet the closed store, and is then lost like any failed record write. - An offline command such as `provision-admin` wires no audit writer, so it records nothing. - Under a long overload only 64 queue-full records are written at a time, and drops past that are not recorded. -- A lock notice lost this way still leaves its `auth.lock_notice` row reading `mailed: true`, so - the throttle holds back the next one. +- A lock notice whose send fails still leaves its `auth.lock_notice` row reading `mailed: true`, + because the send fails on the relay's own task after that row is written. So the throttle holds + back the next lock notice of the kind for the rest of its 24 hours. The failure is on the + `auth.lock_notice_undelivered` row, which the throttle does not read. *Kept as a residual (vault + BACKLOG #3120): closing it would need that row to carry the lock kind and the throttle to read a + second action.* **Residual: the background task adds one way to lose a notice** (BACKLOG #2216). At shutdown the -engine waits about two seconds for pending notices. A notice still pending then is cut off, and may be -neither mailed nor recorded. A failed `auth.lock_notice` row write also changes: it used to fail +engine waits at most two seconds for pending notices, then cancels any still pending, each with one +line. A cancelled notice may be neither mailed nor recorded. The API lifespan's shutdown budget +(BACKLOG #3191) can cut that wait short before it cancels anything, whenever the lock-notice step's +bound falls below those two seconds. A pending notice is then not cancelled and runs on. One that +reaches the security notifier before that notifier begins to close is queued and drained like any +other. One that reaches it later is refused, since nothing would send it (vault BACKLOG #3120), so +no mail goes and its `auth.lock_notice` row says `mailed: false`, which holds back no later notice. +If the store has closed by then, that row write fails and is logged. The notice still went +unmailed, and a missing `mailed: false` row costs nothing at the next lock. A failed `auth.lock_notice` row write also changes: it used to fail the request, and now it is logged instead. The mail goes before its row, so that failure costs at most a duplicate mail at the next lock. Each such line names no account and no lock. It still tells a `logs:view` reader when some notice failed. +**Declined, each with its residual (vault BACKLOG #3120).** + +- **Two mails across engine shards.** The per-account queue and the one-read-at-a-time limit are + each per API process. Engine shards serving their own API ports would each keep their own, so two + locks of one kind landing on two of them at once could each read the throttle before the other + wrote its row, and the owner would get two mails. A duplicate mail is the cheap failure here. +- **The timing of the failure lines.** The paragraph above states it: a line that names nothing + still tells a `logs:view` reader, by when it appears, that some lock's notice failed. +- **No cap on pending notice tasks.** Each lock that lands with a relay wired starts one task, and + nothing bounds how many wait. A task follows a lock, not an attempt, and a lock takes + `lockout_threshold` failures on its counter, so the count grows with the accounts being locked at + once rather than with the attempts against them. Under locks on many accounts the + tasks wait behind the one-at-a-time read, so mails arrive later, and any still pending at + shutdown meet the residual above. A cap would have to drop a notice or run it on + the request path, which is what the task exists to avoid. + **Client attribution ([ADR 0150](adr/0150-client-address-on-audit-entries.md)).** Every row also carries a `client` column — the caller's network address, stamped at write time from the request via the same `client_ip()` the new-client-IP risk signal uses (so behind a declared `trusted_proxies` diff --git a/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md b/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md new file mode 100644 index 000000000..76d1c1ca9 --- /dev/null +++ b/docs/adr/0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md @@ -0,0 +1,112 @@ +# 0210 -- Session supersession follows a time-bounded retired-hash lineage + +- **Status:** Accepted (2026-10-17, on build; the design was set by the batch 257 Manager's brief; + the owner may overrule at review) +- **Date:** 2026-10-17 +- **Related:** vault BACKLOG #3028 (the case this closes), #2146 (engine PR 2028, the atomic + supersession it extends), #3029 (the same race on `logout` and `revoke_own_session`, not built + here), #2592 (the session integrity seal), #1146 (ASVS 7.2.4), ADR 0142 (federated sign-in), + CLAUDE.md section 0 (severity is conditional: there are no deployments) + +Built with the change, in the same pull request. + +## Context + +A console sign-in ends the session its browser presented (ASVS 7.2.4): the new `Set-Cookie` +replaces the old one, so the server is what strands it. Every elevation (MFA, a step-up, a passkey +ceremony, a password re-check) re-keys the session row to a new token hash in place, and the old +hash stops naming any row. + +Engine PR 2028 (BACKLOG #2146) made the supersession's read and revoke one store operation, so an +elevation that starts after it fails closed. It left an elevation that FINISHED before it open. The +presented hash then named nothing, the supersession ended nothing, and the session lived on under +its new token. The window is the whole sign-in, and on the federated leg it includes the identity +provider's round trip, because the start leg reads the cookie (the callback is cross-site and the +Strict cookie never reaches it). Nothing recorded which hash replaced which. + +The constraint the fix must hold is the row's closing item 2: a retired hash must authenticate +nothing, and be usable for nothing but the lookup that ends its successor. + +## Decision + +**A rotation records a link from the hash it retired to its successor, and the login supersession, +and only it, follows those links for a bounded time to the live end and revokes that.** + +1. **A separate table, `session_lineage`** (`retired_hash` primary key, `successor_hash`, + `retired_at`, an index on `retired_at`), on SQLite, Postgres and SQL Server. `rotate_session` + writes one row in the SAME transaction as the re-key, and only when the re-key landed. +2. **The walk is one shared function**, `store/session_lineage.py`'s `revoke_through_lineage`. Each + step tries to revoke the hash it holds; only when that ends nothing does it read the hash's + successor and step. Every backend runs it inside one transaction on one connection. SQLite's + first `UPDATE` takes the database write lock even when it matches nothing, so nothing else can + re-key a row until the commit. On Postgres (READ COMMITTED) and SQL Server each statement reads + what has committed when it runs, so a rotation that commits mid-walk costs one more step, never + the revoke: its link was written in its own transaction. The walk follows at most 32 links. +3. **The bound is the caller's, as `lineage_since`**: a link retired before it is not followed. The + store reads no settings. `AuthService.supersession_lineage_seconds` is + `oidc_flow_ttl_seconds` plus 120 seconds, the longest sign-in a rotation can race plus the + callback's own work after it. With no `lineage_since`, `supersede_session` is the plain revoke + it was, which `_end_lapsed_session` keeps. +4. **The session reaper purges links past the window**, in the same pass as expired sessions, + through a new `lineage_seconds` argument to `purge_expired_sessions`. +5. **No session read consults the table.** `get_session`, `list_sessions` and the service's session + loader read `sessions` only. A test pins, per backend, that only `rotate_session`, + `supersede_session` and `purge_expired_sessions` name the table or a statement on it, and that + only `_supersede_session_hash` passes a window. +6. **The links are not sealed** (BACKLOG #2592). A seal stops a database writer forging authority, + and a link carries none: the most a forged link can do is direct a revoke at a live row, and a + revoke needs no key on any backend, so that writer could revoke the row directly. +7. **The audit row names the session that ended**, by the hash it ended under, and adds the + `presented` hash when a link was followed. + +## Acceptance Criteria + +- **AC-1** -- WHEN an elevation has completed on a session before a sign-in that presents the + session's old token supersedes it, THE SYSTEM SHALL end the session under its new token. + -> `tests/test_session_supersession_lineage.py::test_an_elevation_that_finished_before_the_sign_in_does_not_keep_the_session_live` +- **AC-2** -- WHEN several elevations have completed, THE SYSTEM SHALL follow each link to the live + end. + -> `tests/test_session_supersession_lineage.py::test_the_link_is_followed_over_several_elevations` +- **AC-3** -- IF a link was retired before the window, THEN THE SYSTEM SHALL NOT follow it. + -> `tests/test_session_supersession_lineage.py::test_a_link_older_than_the_window_is_not_followed` +- **AC-4** -- THE SYSTEM SHALL authenticate nothing by a retired token, or by a token whose hash a + link names. + -> `tests/test_session_supersession_lineage.py::test_a_retired_token_authenticates_nothing_while_its_link_is_live`, + `tests/test_session_supersession_lineage.py::test_a_token_whose_hash_links_to_a_live_session_authenticates_nothing` +- **AC-5** -- THE SYSTEM SHALL read the lineage only from the supersession, on every backend. + -> `tests/test_session_supersession_lineage.py::test_only_the_rotation_the_supersession_and_the_purge_touch_the_lineage` +- **AC-6** -- WHEN the reaper runs, THE SYSTEM SHALL delete links past the window and keep the rest. + -> `tests/_session_rotation_contract.py` (`assert_session_lineage_contract`, run on all three + backends) + +## Options considered + +1. **A separate retired-hash table, followed only by the supersession, time-bounded.** CHOSEN. +2. **A `previous_hash` column on `sessions`.** Rejected: one column holds one hop, so a second + elevation during one sign-in would lose the first link, and a sealed column would put the + retired hash under the seal and into every session read's row. +3. **Keep the old hash resolving, revoked, as a tombstone row.** Rejected: a retired hash would + then name a row that every session reader sees, which is the wrong side of closing item 2. +4. **Bound the walk by the sign-in's own start time** instead of a fixed window. Rejected for now: + the password and Kerberos legs would need their start time carried to the mint, and the + Manager's brief set a fixed window of at least the flow TTL. It would narrow the second-holder + case below, and stays open as a refinement. + +## Consequences + +**Positive** -- The case engine PR 2028 left open is closed for any elevation that finished inside +the window, including several in a row. The walk is one function with a `lineage_since` argument, +so `logout` and `revoke_own_session` (#3029) can follow the same links by passing a window. + +**Negative** -- For the length of the window, a retired token gains one power: presented at a +sign-in that succeeds, it ends the session it became. It cannot open or extend one. +`docs/SECURITY.md` states this. Each rotation writes one more row, which the reaper deletes. + +**Stated limits** -- An elevation that finished longer ago than the window, such as one a second +holder of the token completed earlier, is not followed, and neither is a sign-in that runs past the +window; that session lives on under its new token until it expires. More than 32 links in one +window ends nothing. `retired_at` and the bound are both wall-clock readings, so a clock step +between the elevation and the sign-in moves the bound by the step, either way. The window adds the +flow TTL even where federated sign-in is off, where every leg is one request; the brief fixed the +window at no less than the flow TTL, and a narrower one there is open as a refinement. The +Postgres and SQL Server legs run only on hosted runners. diff --git a/docs/adr/README.md b/docs/adr/README.md index b1dc69b97..8d82f62e0 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -232,6 +232,7 @@ what is withheld and what you can request. | [0207](0207-a-single-use-enrolment-code-for-a-directory-account-s-first-engine-factor.md) | **A single-use enrolment code for a directory account's first engine factor** (vault BACKLOG #2609, closing item 2; the decision ADR 0142 Amendment C left open) -- a directory account's first engine factor is bound only with a single-use enrolment code an Administrator issues and hands over out of band. Required at the step that binds and stamps (`confirm_mfa_enrollment`, `finish_webauthn_registration`) when the account is `ad`, holds no engine factor, and the session is not stamped MFA-verified: so every Kerberos session and a federated one with the claim gate off, never a claim-gated federated one. That step also re-checks that the account still holds no factor, because the web console's passkey finish route does not, and the TOTP enable turns on only the secret that was verified, since a later begin replaces the staged one today. The passkey finish route gains the per-actor attempt budget the TOTP confirm routes already draw. Disabling the account clears an outstanding code, and every refused code gets one client-facing reply. The handover is the control, so it must leave the directory: an Administrator checks who is asking and never replies through the account's own mail or chat. The rule reads no session mechanism and no `kerberos_enabled`, and holds with `require_mfa` on or off. At least 128 bits, stored as its SHA-256 digest, one per account, 24 hours, spent by one conditional write before the session is stamped. Issued by an Administrator behind step-up, never for the caller's own account, also by an MFA reset of a directory account, and, for a site where no Administrator can issue one, by a host command on `admin-unlock`'s gate, which is not BACKLOG #2711's outage recovery. Rejected: password only, a TOTP-on-directory-password shortcut, an emailed link, a short Argon2 code, gating on the mechanism | **Proposed (2026-10-05)** -- doc-only, no code; drafted on the owner's direction. Merging records it as Proposed and accepts nothing; Accepted needs the owner. Open: the code with `require_mfa` off, a fixed lifetime, a code from an MFA reset, and the host command's reach. Severity conditional per CLAUDE.md section 0 | | [0208](0208-a-theia-desktop-app-gives-interface-analysts-a-simpler-steps-only-editor.md) | **A Theia desktop app gives interface analysts a simpler Steps-only editor** (owner rulings and Manager decisions of 2026-10-07; rewrites the uncommitted 2026-10-02 hosted-Theia drafts after two four-lens reviews) -- two Electron builds sharing one native Theia Steps extension: an analyst build capped at the Steps level, and a developer build that is a full IDE with a Steps / Split / Code switch and no per-user limits. The purpose is a simpler interface, not enforcement. The analyst build signs in to the engine, clears every session gate, proves the session on a non-exempt route at start-up and before each save, and turns on editing only for `code:steps` (new, advisory, read by the editor only) or `code:edit`. That role check is the primary control and a guardrail: the developer build, VS Code and any other tool are outside it. An optional repository check classifies a change as Steps-only by comparing `code` rows, every control header and each typed row's full statement against base, and passes anything else only with a `code:edit` reviewer's approval of the head commit; it holds against a deliberate change only when run from a definition the change cannot edit. Analyst Test takes a generator spec and never passes `--show-phi`. Review R1 is a precondition answered by the R1 fix and a typed-only `lens rewrite` mode the analyst build always sets; both landed in PR 2155 (2026-10-08). Analyst is a built-in role. The analyst build drops three ADR 0076 guardrails (Amendment G). Hosting is deferred to its own ADR. Rejected: a filtered `@theia/plugin-ext` analyst build, per-user limits in the developer build, VS Code for analysts, a Steps editor in the web console. Spec: [`docs/design/theia-analyst-editor.md`](../design/theia-analyst-editor.md) | **Proposed (2026-10-07)** -- doc-only, no editor code. Merging records it as Proposed and accepts nothing; Accepted needs the owner. Licence AGPL-3.0-or-later (owner ruling). Open: spikes S-1 to S-4 (S-1 is met except its Test and language-server legs, measured on a browser target; S-2 is met on the browser build and on Electron, spike S-2b; S-4's classifier half is met by a prototype and its base-ref workflow half is untested), the start-up probe route (an implementation detail), and the differences ADR 0076 Amendment G, G.7, records between PR 2155's lens code and its rules | | [0209](0209-mllp-and-tcp-deliveries-refuse-a-payload-holding-more-than-one-msh.md) | **MLLP and TCP deliveries refuse a payload holding more than one MSH** (vault BACKLOG #2858; the outbound decision ADR 0206 rule 5 deferred) -- an MLLP or TCP delivery refuses a payload that opens with `MSH` and holds a second `MSH` line, CR or LF line ends alike, as `NegativeAckError(permanent=True)` with code `multiple-messages` and content-free text. The check sits in `check_frame_bytes`, so a live send, each batch member (judged as stored) and a shadow outbound give one answer; the ingress guard's reading moves to `parsing/split.py`, beside the delivery's, and a test holds the two to one answer. No batch-sender opt-out: ADR 0082's `BHS` envelope is the batch path and is never read, nor is an X12 relay or any payload not led by `MSH`. HTTP and File outbounds are out of scope. Rejected: a declared batch sender, splitting into N frames, refusing on every outbound, a `check_frame`-only check | **Accepted (2026-10-17, on build)** -- decision delegated by the owner under the driver rule and taken by the batch 265 Manager; the owner may overrule at review. Built with the change. Severity conditional per CLAUDE.md section 0 | +| [0210](0210-session-supersession-follows-a-time-bounded-retired-hash-lineage.md) | **Session supersession follows a time-bounded retired-hash lineage** (vault BACKLOG #3028; extends #2146, engine PR 2028) -- each session rotation writes a `session_lineage` link (retired hash, successor hash, `retired_at`) in its own transaction, on SQLite, Postgres and SQL Server. The login supersession, and nothing else, follows those links over any number of hops (at most 32) to the live end and revokes it in the same transaction, for `oidc_flow_ttl_seconds` plus 120 seconds after each rotation; the session reaper purges older links. No session read consults the table, so a retired hash authenticates nothing; a test pins the readers per backend. The links are unsealed, because a forged one can only direct a revoke, which needs no key. One shared walk, so #3029 can reuse it. Rejected: a `previous_hash` column, a revoked tombstone row, a per-sign-in start-time bound. Open: an elevation older than the window, such as a second holder's, is not followed | **Accepted (2026-10-17, on build)** -- design set by the batch 257 Manager's brief; the owner may overrule at review. Severity conditional per CLAUDE.md section 0 | ## Retired allocations — numbers taken but never authored diff --git a/messagefoundry/__main__.py b/messagefoundry/__main__.py index 194de79a3..fd5f1d600 100644 --- a/messagefoundry/__main__.py +++ b/messagefoundry/__main__.py @@ -8716,12 +8716,13 @@ async def notice() -> str: return "no_channel" notifier.start() try: - await notifier.notify( + queued = await notifier.notify( SecurityEvent(event_type=MFA_ENABLED, username=user.username, email=address) ) finally: await notifier.aclose() - return "dispatched" + # What the notifier said it took (vault BACKLOG #3120), never a forecast. + return "dispatched" if queued else "no_channel" try: report["holder_notice"] = run_guarded(notice()) diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 00b89f482..69e1b0bfc 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -287,6 +287,7 @@ from messagefoundry.auth.audit_visibility import reads_audit_copies_in_the_log from messagefoundry.auth.reconcile import ( HOLD_REASON, + MISCONFIGURED_ABORT, PAGING_CAP, PAGING_DEADLINE, PROBE_FAULT_ABORT, @@ -1350,7 +1351,9 @@ async def _outliving(bound: float) -> None: if auth is not None and owns_auth: # close_background is the shutdown call the service names, with its own bound (its drain # and cancel grace, in auth/service.py). Only a bound below that cuts it before it cancels - # its notices itself; a notice cut that way meets the closed store and logs its own failure. + # its notices itself. A notice cut that way runs on: the security notifier queues it if it + # arrives before that notifier begins to close, and refuses it after, so its row reads + # mailed: false, or meets the closed store and logs its own failure (vault BACKLOG #3120). steps.append( _TeardownStep("the lock-notice drain", 1.0, lambda _b: auth.close_background()) ) @@ -10192,13 +10195,16 @@ async def _session_reaper( store: Store, *, idle_seconds: float | None = None, + lineage_seconds: float | None = None, wall: Callable[[], float] | None = None, mono: Callable[[], float] | None = None, ) -> None: """Drop expired session rows, first ``_SESSION_REAP_FIRST_DELAY`` after start and then every ``_SESSION_REAP_INTERVAL``, until the task is cancelled. With ``idle_seconds`` given, idle-expired rows go too (BACKLOG #2096); the lifespan passes the idle timeout the validator - uses. + uses. With ``lineage_seconds`` given, session lineage links older than that go too (vault + BACKLOG #3028); the lifespan passes the window the supersession follows them for. Purging as of + the earlier instant below only keeps a link longer, never shortens that window. **The reaper skips a pass when the wall clock has jumped forward (BACKLOG #2283).** The purge judges expiry and idleness by the wall clock, and its deletes cannot be undone. Run just after @@ -10257,7 +10263,9 @@ def _clocks() -> tuple[float, float]: continue try: await store.purge_expired_sessions( - now=now - _SESSION_REAP_STEP_TOLERANCE, idle_seconds=idle_seconds + now=now - _SESSION_REAP_STEP_TOLERANCE, + idle_seconds=idle_seconds, + lineage_seconds=lineage_seconds, ) except asyncio.CancelledError: raise @@ -10337,6 +10345,9 @@ async def _directory_reconciler( #: The source label of the probe fault's own ``ad_reconcile_aborted`` instance (BACKLOG #3100), apart #: from the breaker's and the referral's for the same reason. _PROBE_FAULT_ALERT_SOURCE = "directory-reconciler-probe-fault" +#: The source label of the configuration refusal's own ``ad_reconcile_aborted`` instance (BACKLOG +#: #3010), apart from the others for the same reasons. +_MISCONFIGURED_ALERT_SOURCE = "directory-reconciler-misconfigured" #: The source label prefix of a paging refusal's ``ad_reconcile_aborted`` instance (vault BACKLOG #: #3696). The account name follows it, so each account is its own instance and throttle, as #: ``ad_session_revoked`` is keyed on the account: a second account entering inside the first's @@ -10382,22 +10393,38 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin alert with reason ``probe_fault``, under its own source label (:data:`_PROBE_FAULT_ALERT_SOURCE`). It has no inverse: an operator resolves it. + A search the directory refused for its configuration (BACKLOG #3010) is not an outage either: a + search base that was deleted or moved, or that the service account may not read, ends every + search with noSuchObject, invalidDNSyntax or insufficientAccessRights on every pass. Any pass + with such a refusal is ``auth.ad_reconcile_misconfigured`` and becomes one + ``ad_reconcile_aborted`` alert with reason ``directory_misconfigured``, under its own source + label (:data:`_MISCONFIGURED_ALERT_SOURCE`). Like the fault's, it has no inverse. Before #3010 + such a base read as "not found" for every account, so it tripped the breaker on an estate + large enough; read as an outage, it would page nobody. + A paging refusal (vault BACKLOG #3696) pages once per account as it enters that state, after the revocations and on an outage-shaped pass too, matching its ``auth.ad_reconcile_paging_refused`` row; - :func:`_alert_paging_refusals` says how. + :func:`_alert_paging_refusals` says how. It is a bound the engine set, on an account the + directory answered for, so it is not the configuration refusal above, which is a result code + the directory returned; the two never describe one probe. + + The breaker's alert counts the accounts the pass judged, its denominator, and so does each + paging refusal's. The referral's, the refusal's and the fault's count the probes the pass + asked, which leaves out an id-less row (BACKLOG #3170): only a probe that was sent can be + referred, refused or raise a fault. Reads the RETURNED plan, so it sees only a pass that finished. A failed write of the pass's own - held, aborted, skipped, referred, faulted or unkeyed-binding row no longer ends the pass - (BACKLOG #2137): the service logs it at ERROR and returns the plan, so these alerts still fire. - Since BACKLOG #3018 the same holds for a revocation's own ``auth.ad_session_revoked`` row and a - rename's rows. A pass that still raises part-way, such as on a store fault ending one account's - sessions, returns no plan, so no alert is raised for the revocations it applied before that - point. Their rows stand, except a row that was refused: that account has neither its row nor - its alert. ``AuthService._audit_reconciler_row`` says which errors are soft. - - Each standing alert but the probe fault's and a paging refusal's also gets its inverse, which - pages nobody and resolves the open instance (BACKLOG #2136): ``ad_reconcile_breaker_cleared`` - and ``ad_reconcile_hold_released``, and + held, aborted, skipped, referred, misconfigured, faulted or unkeyed-binding row no longer ends + the pass (BACKLOG #2137): the service logs it at ERROR and returns the plan, so these alerts + still fire. Since BACKLOG #3018 the same holds for a revocation's own ``auth.ad_session_revoked`` + row and a rename's rows. A pass that still raises part-way, such as on a store fault ending one + account's sessions, returns no plan, so no alert is raised for the revocations it applied before + that point. Their rows stand, except a row that was refused: that account has neither its row + nor its alert. ``AuthService._audit_reconciler_row`` says which errors are soft. + + Each standing alert but the probe fault's, the configuration refusal's and a paging refusal's + also gets its inverse, which pages nobody and resolves the open instance (BACKLOG #2136): + ``ad_reconcile_breaker_cleared`` and ``ad_reconcile_hold_released``, and ``ad_reconcile_breaker_cleared`` under the referral's source for the referral's instance. The auth service decides when a pass is evidence of a clear (``plan.breaker_clear``, ``plan.hold_clear``, ``plan.referral_clear``; see ``AuthService._mark_reconcile_clears``), so @@ -10412,9 +10439,9 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin # BACKLOG #3696). An outage applies no revocation, so nothing here pages ahead of it. _alert_paging_refusals(plan, sink) return - if plan.aborted is not None and not plan.directory_referral and not plan.probe_fault: - # A breaker trip. A pass of referrals only is not one: its one alert is the referral's below. - # Nor is a pass of faults only (BACKLOG #3100): its one alert is the fault's below. + if plan.aborted is not None and not plan.judged_nothing: + # A breaker trip. A pass that judged nothing is not one: its alerts are the referral's, the + # refusal's or the fault's below (BACKLOG #2538, #3010, #3100), and an outage has none. sink.ad_reconcile_aborted( "directory-reconciler", reason=plan.aborted, @@ -10432,17 +10459,24 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin sink.ad_reconcile_aborted( _REFERRAL_ALERT_SOURCE, reason=REFERRAL_ABORT, - # The probes ASKED (BACKLOG #3170): an UNKEYED row was never sent, so no referral - # could answer it. The breaker's count above stays its judged denominator. + # The probes ASKED, as the refusal's and the fault's below (BACKLOG #3170). probed=plan.asked, detail=auth.directory_reconcile_referral or REFERRAL_ABORT, ) + if plan.misconfigured: + # BACKLOG #3010. Beside the referral's, in the same order the service records them. + sink.ad_reconcile_aborted( + _MISCONFIGURED_ALERT_SOURCE, + reason=MISCONFIGURED_ABORT, + probed=plan.asked, + detail=auth.directory_reconcile_misconfigured or MISCONFIGURED_ABORT, + ) if plan.faulted: - # BACKLOG #3100. Beside the referral's, in the same order the service records them. + # BACKLOG #3100. sink.ad_reconcile_aborted( _PROBE_FAULT_ALERT_SOURCE, reason=PROBE_FAULT_ABORT, - probed=plan.probed, + probed=plan.asked, detail=auth.directory_reconcile_fault or PROBE_FAULT_ABORT, ) if plan.hold: @@ -11590,7 +11624,11 @@ async def _audit_upload_prune(meta: UploadedFileMeta) -> None: auth_settings.oidc_redirect_path, ) reaper = asyncio.create_task( - _session_reaper(store, idle_seconds=auth.session_idle_seconds) + _session_reaper( + store, + idle_seconds=auth.session_idle_seconds, + lineage_seconds=auth.supersession_lineage_seconds, + ) ) if _initial_credential_warn_lead(auth) is not None: # BACKLOG #1141 (ASVS 6.4.5): a nudge for every unclaimed temporary password an diff --git a/messagefoundry/api/approvals.py b/messagefoundry/api/approvals.py index 290066ab9..89241e45a 100644 --- a/messagefoundry/api/approvals.py +++ b/messagefoundry/api/approvals.py @@ -105,12 +105,14 @@ } -#: How long :meth:`ApprovalGate.drain` waits for the gate's writes still running at shutdown, at -#: least the outcome writes and the request writes (vault BACKLOG #3193). It shares NSSM's 15 s -#: graceful-stop window (AppStopMethodConsole in scripts/service/install-service.ps1) with the -#: upload runner's 5 s stop, which runs first, and engine.stop() still has to run after it. Each -#: write is a status change or a new row with its audit row, plus at most a read-back, so a few -#: seconds is ample. +#: The default wait of :meth:`ApprovalGate.drain`, for a caller that passes no timeout. The drain +#: waits for the gate's writes still running, at least the outcome writes and the request writes +#: (vault BACKLOG #3193). The API lifespans do not use this default: they pass a share of the drain +#: step's bound, which can be longer or shorter. ``_SHUTDOWN_BUDGET_SECONDS`` in ``api/app.py`` +#: states the budget and the graceful-stop window arithmetic (BACKLOG #3191); ``_app_drain_steps`` +#: and the managed lifespan's teardown hold the steps and their order. +#: Each write is a status change or a new row with its audit row, plus at most a read-back, so a +#: few seconds is ample. DRAIN_TIMEOUT_SECONDS = 3.0 diff --git a/messagefoundry/auth/ldap.py b/messagefoundry/auth/ldap.py index c1db8e337..af3b239dd 100644 --- a/messagefoundry/auth/ldap.py +++ b/messagefoundry/auth/ldap.py @@ -145,6 +145,15 @@ def __init__( self.code = code self.code_name = code_name + @property + def configuration(self) -> bool: + """Whether the code points at the configuration rather than at a passing load + (:data:`_CONFIGURATION_CODES`): noSuchObject (32), invalidDNSyntax (34) and + insufficientAccessRights (50). Such a code recurs on every attempt until the configuration + changes, so the session reconciler reads it as misconfigured, which alerts, and not as + unavailable, which pages nobody (BACKLOG #3010).""" + return self.code in _CONFIGURATION_CODES + class LdapUndecodableError(LdapError): """ldap3 could not decode the directory's response to an operation (BACKLOG #3010). @@ -808,6 +817,11 @@ def _referral_error(operation: str, message: str) -> LdapReferralError: 50: " insufficientAccessRights means the service account may not read it.", } +#: The codes that point at the configuration rather than at a passing load +#: (:attr:`LdapUnansweredError.configuration`). Kept apart from the hints on purpose: a hint added +#: for a transient code must not make it page as a misconfiguration. +_CONFIGURATION_CODES = frozenset({32, 34, 50}) + def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | None = None) -> None: """Raise :class:`LdapReferralError` for a referral, and otherwise :class:`LdapUnansweredError` @@ -832,18 +846,17 @@ def _refuse_unanswered(conn: Any, operation: str, *, answers: frozenset[int] | N read as absence it would read every account absent at once, which the reconciler's mass-revoke breaker does not stop at or under its floor. - **What each caller makes of it.** The session reconciler reads an :class:`LdapError` as - unavailable, which never strikes; a sign-in writes ``auth.login_error``; the nested group search - raising means no caller sees a shorter group list. + **What each caller makes of it.** The session reconciler reads one of the configuration codes + (:attr:`LdapUnansweredError.configuration`) as misconfigured and any other as unavailable. + Neither strikes. A sign-in writes ``auth.login_error``; the nested group search raising means + no caller sees a shorter group list. - **The cost, named.** Unavailable never revokes, so a code that recurs on every pass, such as a - deleted search base, stops offboarding and the role re-diff for as long as it lasts, and nothing - latches an alert for it. Making a standing one visible is #3010's item 4, which the batch - deferred to vault BACKLOG #3100's item 3. As filed, that item names the reconciler's broad - ``except Exception`` arm, which this error does not reach, so it must widen to cover the - ``except LdapError`` arm before the deferral holds. This warns once per operation and code, and - the text carries no directory text: the code, ldap3's name for it and the operation, never the - diagnostic message the directory sent. + **The cost, named.** Neither reading revokes, so a code that recurs on every pass, such as a + deleted search base, stops offboarding and the role re-diff for as long as it lasts. A + configuration code alerts on every pass it answers, and a pass where nothing else answered + aborts as ``directory_misconfigured``. Any other code reads as an outage, which pages nobody. + This warns once per operation and code, and the text carries no directory text: the code, + ldap3's name for it and the operation, never the diagnostic message the directory sent. **A result that is not a dict is let through.** ldap3's synchronous strategy, the only one built here, reads ``self.result['type']`` straight after every search, so a real search cannot return @@ -878,9 +891,16 @@ def _unanswered(operation: str, code: int | None) -> LdapUnansweredError: else: name = RESULT_CODES.get(code, "an unknown result code") shown, hint = f"result code {code} ({name})", _UNANSWERED_HINTS.get(code, "") + # A configuration code is not read as the directory not answering: the session reconciler + # reads it as a configuration refusal, which alerts (LdapUnansweredError.configuration). + reading = ( + "a fault in the configuration" + if code in _CONFIGURATION_CODES + else "the directory not answering" + ) message = ( f"AD ended the {operation} with {shown}, which is not an answer; the engine reads it as " - f"the directory not answering (BACKLOG #3010).{hint}" + f"{reading} (BACKLOG #3010).{hint}" ) # Keyed on ldap3's name, not the raw code: a directory chooses the code, and ldap3's table is # finite, so the set of warnings stays bounded. diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index 323fbf3d6..7cb61ee09 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -129,11 +129,14 @@ #: one, so a line per undelivered lock notice told that reader which candidate was right. #: #: Only ``ACCOUNT_LOCKED`` carries that bit. Every other kind is sent on an authenticated action, a -#: completed sign-in, or a clock, so it keeps its per-event line (BACKLOG #1139). A lock notice with -#: no relay or no address is still recorded where only an administrator reads it, as ``mailed: false`` -#: on the ``auth.lock_notice`` row. A full queue or a failed send is recorded on its own -#: ``auth.lock_notice_undelivered`` row (BACKLOG #2383), since the ``auth.lock_notice`` row is written -#: at the hand-off, as ``mailed: true``. An instance with no relay at all is reported at startup by +#: completed sign-in, or a clock, so it keeps its per-event line (BACKLOG #1139). A lock notice the +#: notifier did not take, for at least no relay, no address, no room on its queue or a notifier +#: closing at shutdown, is still recorded where only an administrator reads it, as +#: ``mailed: false`` on the ``auth.lock_notice`` row (vault BACKLOG #3120 for the last two). A full +#: queue, a closing notifier or a failed send is also recorded on its own +#: ``auth.lock_notice_undelivered`` row (BACKLOG #2383). A failed send happens after the +#: ``auth.lock_notice`` row is written at the hand-off, so that row still reads ``mailed: true``; the +#: residual is stated in ``docs/SECURITY.md``. An instance with no relay at all is reported at startup by #: the serve gate, unless enforcement is ``warn`` and the operator waived notices in writing. LOG_SILENT_EVENT_TYPES: frozenset[str] = frozenset({ACCOUNT_LOCKED}) @@ -233,9 +236,17 @@ class SecurityEvent: class SecurityNotifier(Protocol): """Push a security event to the affected user out-of-band. Implemented in ``pipeline/`` (email over the ``[alerts]`` SMTP transport) and injected into :class:`AuthService`. Must be best-effort and - must not raise into the auth path (the caller still guards it).""" + must not raise into the auth path (the caller still guards it). - async def notify(self, event: SecurityEvent) -> None: ... + ``notify`` returns whether it took the event for sending: ``False`` when it dropped the event + on the spot, for example for want of an address or room on its queue, or because it is closing + or closed and would never send it (vault BACKLOG #3120). An implementation that can be closed + answers ``False`` from the start of its close. + ``True`` is a hand-off, never a delivery receipt: a send that fails later is not reported back. + The ``auth.lock_notice`` row's ``mailed`` field is recorded from this, so a notice dropped here + does not hold the next lock's notice back for a day.""" + + async def notify(self, event: SecurityEvent) -> bool: ... def deadline_utc(ts: float) -> str | None: diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index 9f50743af..698508364 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -24,25 +24,36 @@ at once. :func:`hold_engaged` holds those accounts, and only those: none of them is revoked, and the rest of the estate is reconciled as usual. The mass-revoke breaker below cannot do this on a small estate, because its absolute floor lets five or fewer revocations through. -4. **A mass-revoke circuit breaker.** A misconfigured search base or a moved OU returns "not found" - for **every** user, which looks the same as "everyone was deleted". :func:`breaker_tripped` - aborts such a pass wholesale rather than signing out the estate. This is why the pass is - planned in full before anything is written: an abort must leave the store byte-identical, - including the role re-diff. +4. **A mass-revoke circuit breaker.** Users moved out of the search base, a base that exists but + is the wrong one, or a service account that lost read rights on the user entries but not on + the base, return "not found" for **every** user, which looks the same as "everyone was + deleted". :func:`breaker_tripped` aborts such a pass wholesale rather than signing out the + estate. This is why the pass is planned in full before anything is written: an abort must leave + the store byte-identical, including the role re-diff. A base that was deleted or moved, or that + the service account may not read, is not "not found": the search ends with a result code, which + reads :attr:`ProbeOutcome.MISCONFIGURED` (BACKLOG #3010). A referral is not an outage, and not a reason to stop the pass either (BACKLOG #2538). A :attr:`ProbeOutcome.REFERRED` account is left unjudged, like an unreachable one, and every other -probe in the pass is judged as usual. Unlike an outage, a referral alerts. A pass whose only answers are -referrals and failures aborts as ``directory_referral``, which judges no account it asked about -and alerts. Such a pass, like an outage, still strikes an UNKEYED row (BACKLOG #3170). +probe in the pass is judged as usual. Unlike an outage, a referral alerts. A pass whose every answer +went unjudged, at least one a referral, aborts as ``directory_referral``, which judges no account it +asked about and alerts. Such a pass, like an outage, still strikes an UNKEYED row (BACKLOG #3170). A probe fault is not an outage either (BACKLOG #3100). A :attr:`ProbeOutcome.FAULT` account, whose probe raised something that is not an LDAP error, or a response ldap3 could not decode, is left -unjudged like an unreachable one, and -unlike one it alerts on every pass that probes it. A pass whose only answers are faults and failures, -at least one a fault, aborts as ``probe_fault``, which judges no account it asked about and alerts. +unjudged like an unreachable one, and unlike one it alerts on every pass that probes it. A pass +whose only answers are faults and failures, at least one a fault, aborts as ``probe_fault``, which +judges no account it asked about and alerts. +A pass where every probe fails and none faults is a plain outage, which pages nobody. It too still strikes an UNKEYED row (BACKLOG #3170). +A search the directory refused for a reason that points at the configuration is not an outage +either (BACKLOG #3010). A :attr:`ProbeOutcome.MISCONFIGURED` account is left unjudged like an +unreachable one, and unlike one it alerts on every pass that probes it. A pass whose only answers +are those refusals, faults and failures, at least one such refusal and no referral, aborts as +``directory_misconfigured``, which judges no account it asked about, alerts, and still strikes an +UNKEYED row. + The pass also re-diffs channel scope (ADR 0198, BACKLOG #1957). It revokes when the directory would withdraw or narrow a principal's scope, and it never writes the scope: the next login does, through the same :func:`~messagefoundry.auth.channel_scope.decide_ad_channel_scope` this module plans with. @@ -77,18 +88,20 @@ class ProbeOutcome(Enum): #: (BACKLOG #1639). A single one beside readable answers strikes and revokes like :attr:`ABSENT`; #: a wave of them is held (ADR 0195, :func:`hold_engaged`). UNDETERMINED = "undetermined" - #: The lookup matched nothing, or an id-keyed probe held a stored ``objectGUID`` that could not - #: be parsed, so no search ran, or an id-keyed search's entry did not read back that id (BACKLOG - #: #2027). **Ambiguous**: deleted, moved out of the search base, or a search - #: base that was never right. Strikes, never revokes on its own. Since ADR 0195 a disabled entry, + #: The lookup ended with success and matched nothing, or an id-keyed probe held a stored + #: ``objectGUID`` that could not be parsed, so no search ran, or an id-keyed search's entry did + #: not read back that id (BACKLOG #2027). **Ambiguous**: deleted, moved out of the search base, + #: or a search base that exists but was never the right one. A base that does not exist, or that + #: may not be read, is not this: the search ends with a result code, which is + #: :attr:`MISCONFIGURED` (BACKLOG #3010). Strikes, never revokes on its own. Since ADR 0195 a disabled entry, #: or one whose ``userAccountControl`` is unreadable, is NOT this outcome, with one exception #: (BACKLOG #2027): an entry an id-keyed search found that does not read back that id is this #: outcome whatever its account state, because it is not this account's entry. ABSENT = "absent" #: The directory could not be consulted (``LdapError`` — connectivity/bind/config — other than a - #: referral or a response ldap3 could not decode), or a network-class ``OSError`` escaped the - #: directory layer (BACKLOG #3100). - #: Contributes nothing: no strike, no revocation, no strike reset. + #: referral, a configuration code or a response ldap3 could not decode), or a network-class + #: ``OSError`` escaped the directory layer (BACKLOG #3100). Contributes nothing: no strike, no + #: revocation, no strike reset. UNAVAILABLE = "unavailable" #: The directory answered with a referral, which the engine never follows (``LdapReferralError``, #: BACKLOG #2538). Contributes nothing for this account, like :attr:`UNAVAILABLE`: no strike, @@ -110,6 +123,16 @@ class ProbeOutcome(Enum): #: would leave that account unrevoked without a page. The value is the certificate path's #: ``probe_fault`` refusal reason too. FAULT = "probe_fault" + #: The directory refused the search with a result code that points at the configuration rather + #: than at a passing load: noSuchObject, invalidDNSyntax or insufficientAccessRights + #: (``LdapUnansweredError.configuration``, BACKLOG #3010). A deleted or moved search base, or one + #: the service account may not read, answers so for every account. Contributes nothing for this + #: account, like :attr:`UNAVAILABLE`: no strike, no revocation, no strike reset, no recorded + #: outcome, and it never moves the ADR 0195 hold. Unlike it, it alerts, because it recurs on + #: every pass until the configuration is fixed: read as unavailable, it would stop revocation + #: without a page. Before BACKLOG #3010 such a search read as "not found", so a deleted base + #: struck every account and could trip the breaker. + MISCONFIGURED = "misconfigured" #: The row carries no ``directory_object_id``, so it was NOT probed (BACKLOG #2434). A name #: probe could read another account's entry. No sign-in or step-up admits such a row, so a #: session it holds is anomalous. Plans exactly as :attr:`ABSENT` does: it strikes, and revokes @@ -136,7 +159,9 @@ class Probe: #: For :attr:`ProbeOutcome.REFERRED`: the refusal's text, which names the referred search and #: the referred hosts and nothing else (BACKLOG #2530). The pass logs the first one. For #: :attr:`ProbeOutcome.FAULT`: the exception's type name, never its message (BACKLOG #3100). - #: For :attr:`ProbeOutcome.UNAVAILABLE`: :data:`PAGING_DEADLINE` or :data:`PAGING_CAP` when the + #: For :attr:`ProbeOutcome.MISCONFIGURED`: the refusal's text, which names the operation and the + #: result code and never the directory's diagnostic message (BACKLOG #3010). For + #: :attr:`ProbeOutcome.UNAVAILABLE`: :data:`PAGING_DEADLINE` or :data:`PAGING_CAP` when the #: group search was refused for its bounds rather than unanswered (vault BACKLOG #3696), else #: None. None for every other outcome. detail: str | None = None @@ -177,8 +202,10 @@ class UsernameRefresh: #: The closed-set slug the held audit row and alert carry (ADR 0195 rule item 9). HOLD_REASON = "user_account_control_undetermined" -#: The ``aborted`` slug of a pass whose only answers were referrals and failures (BACKLOG #2538). -#: Also the reason slug of the referral's own audit row and alert, on any pass with a referral. +#: The ``aborted`` slug of a pass whose every answer went unjudged, at least one a referral (BACKLOG +#: #2538); the others may be failures, configuration refusals or faults (BACKLOG #3010, #3100), +#: which are then recorded under their own alerts as well. Also the reason slug of the referral's +#: own audit row and alert, on any pass with a referral. REFERRAL_ABORT = "directory_referral" #: The ``aborted`` slug of a pass whose only answers were probe faults and failures, at least one a @@ -186,6 +213,11 @@ class UsernameRefresh: #: with a fault. The same string as :attr:`ProbeOutcome.FAULT`'s value. PROBE_FAULT_ABORT = ProbeOutcome.FAULT.value +#: The ``aborted`` slug of a pass whose only answers were configuration refusals, faults and +#: failures, at least one a configuration refusal and none a referral (BACKLOG #3010). Also the +#: reason slug of that refusal's own audit row and alert, on any pass with one. +MISCONFIGURED_ABORT = "directory_misconfigured" + #: The causes of an :attr:`ProbeOutcome.UNAVAILABLE` probe whose group search the engine refused #: for its own bounds (vault BACKLOG #3696): past ``[auth].ad_group_search_deadline``, or past the #: cap on groups or its page backstop. It pages, once per account as it enters that state @@ -195,7 +227,14 @@ class UsernameRefresh: PAGING_CAP = "group_search_cap" #: The outcomes that judge nothing about the account: no strike, no recorded outcome (rule item 4). -_UNJUDGED = frozenset({ProbeOutcome.UNAVAILABLE, ProbeOutcome.REFERRED, ProbeOutcome.FAULT}) +_UNJUDGED = frozenset( + { + ProbeOutcome.UNAVAILABLE, + ProbeOutcome.REFERRED, + ProbeOutcome.FAULT, + ProbeOutcome.MISCONFIGURED, + } +) @dataclass(frozen=True) @@ -229,21 +268,22 @@ class ReconcilePlan: renames: tuple[UsernameRefresh, ...] = () #: Strikes to record: ``user_id -> consecutive ABSENT, DISABLED, UNDETERMINED or UNKEYED #: count``. A PRESENT probe maps to 0 (reset), and so does a HELD one (ADR 0195 rule item 7; - #: ``held`` tells the two apart). An UNAVAILABLE, REFERRED or FAULT probe is absent from this mapping, - #: leaving whatever strike the user already carried untouched. Populated even on a breaker - #: abort — see :func:`plan_pass`. On an outage or a pass of referrals or faults only it holds - #: the UNKEYED rows' strikes alone (BACKLOG #3170), since those were judged without the - #: directory. + #: ``held`` tells the two apart). An unjudged probe (UNAVAILABLE, REFERRED, FAULT or + #: MISCONFIGURED) is absent from this mapping, leaving whatever strike the user already carried + #: untouched. Populated even on a breaker abort — see :func:`plan_pass`. On a pass that judged + #: nothing (:attr:`judged_nothing`) it holds the UNKEYED rows' strikes alone (BACKLOG #3170), + #: since those were judged without the directory. strikes: Mapping[str, int] = field(default_factory=dict) #: Candidates this pass judged, asked or not. Excludes the per-pass budget remainder. An UNKEYED #: row counts here, because it can revoke. This is the breaker's denominator (``judged``). probed: int = 0 #: Probes this pass actually sent to the directory: ``probed`` less its UNKEYED rows (BACKLOG - #: #2434). An outage and a pass of referrals or faults only are judged over these, and the - #: outage's log line and the ``asked`` field of its ``auth.ad_reconcile_skipped`` row count - #: these. So do the referral's latched message, the ``asked`` field of its - #: ``auth.ad_reconcile_referred`` row and its alert's count (BACKLOG #3170): a referral can - #: only answer a probe that was sent. + #: #2434). Every abort but the breaker's is judged over these. The outage, the referral, the + #: configuration refusal and the probe fault each count these in their log line or latched + #: message and in the ``asked`` field of their audit row, and the last three in their alert's + #: count (BACKLOG #3170, #3010): only a probe that was sent can be referred, refused or raise a fault. + #: The breaker's alert counts ``probed``, its judged denominator, and so does each paging + #: refusal's (vault BACKLOG #3696), which concerns one account and reports the pass's count. asked: int = 0 unavailable: int = 0 #: ``user_id`` of every probe the directory answered with a referral (BACKLOG #2538). Any one @@ -252,14 +292,18 @@ class ReconcilePlan: #: ``user_id`` of every probe that raised a fault (:attr:`ProbeOutcome.FAULT`, BACKLOG #3100). #: Any one alerts, and none of them stops the rest of the pass being judged. faulted: tuple[str, ...] = () + #: ``user_id`` of every probe the directory refused for its configuration + #: (:attr:`ProbeOutcome.MISCONFIGURED`, BACKLOG #3010). Any one alerts, and none of them stops + #: the rest of the pass being judged. + misconfigured: tuple[str, ...] = () #: Non-None when the pass ABORTED and must apply nothing: the breaker tripped, every probe the - #: pass asked failed (a whole-directory outage), every probe it asked failed or was referred, - #: at least one of them referred, or every probe it asked failed or faulted, at least one of - #: them a fault (BACKLOG #3100). An UNKEYED row was not asked, so it never stops either abort - #: (BACKLOG #2434). The value is a closed-set operator-facing slug. + #: pass asked failed (a whole-directory outage), or every probe it asked went unjudged with at + #: least one of them referred, refused for its configuration or faulted, which take the slug in + #: that order (BACKLOG #2538, #3010, #3100). An UNKEYED row was not asked, so it never stops an + #: abort (BACKLOG #2434). The value is a closed-set operator-facing slug. aborted: str | None = None #: Outcomes to record in the caller's per-candidate record (ADR 0195 rule item 4): every probe of - #: this pass except an UNAVAILABLE, REFERRED or FAULT one, which leaves the prior entry in place. + #: this pass except an unjudged one (``_UNJUDGED``), which leaves the prior entry in place. #: Populated on a breaker abort too, like ``strikes``. outcomes: Mapping[str, ProbeOutcome] = field(default_factory=dict) #: Whether this pass held its UNDETERMINED probes (:func:`hold_engaged`). @@ -298,8 +342,8 @@ def directory_outage(self) -> bool: @property def directory_referral(self) -> bool: - """True when the pass aborted because every probe it asked was referred or failed, at least - one of them referred (BACKLOG #2538). It judged nothing, like an outage, but it alerts: a + """True when the pass aborted because every probe it asked went unjudged, at least one of + them referred (BACKLOG #2538). It judged nothing, like an outage, but it alerts: a referral usually means ``ad_user_search_base`` lies in another domain of the forest, which no later pass fixes on its own. A pass with a referral beside other answers is NOT this. It judged those answers, and only ``referred`` says it saw a referral.""" @@ -310,18 +354,33 @@ def probe_fault(self) -> bool: """True when the pass aborted because every probe it asked faulted or failed, at least one of them a fault (BACKLOG #3100). It judged nothing, like an outage, but it alerts: the engine could not read what the directory returned, which no later pass fixes on its own. - A pass with a referral as well is a referral pass. A pass with a fault beside other answers - is NOT this; only ``faulted`` says it saw a fault.""" + A pass with a referral or a configuration refusal as well is one of those passes. A pass + with a fault beside other answers is NOT this; only ``faulted`` says it saw a fault.""" return self.aborted == PROBE_FAULT_ABORT + @property + def directory_misconfigured(self) -> bool: + """True when the pass aborted because every probe it asked was refused for its + configuration, faulted or failed, at least one refused for its configuration and none + referred (BACKLOG #3010). It judged nothing, like an outage, but it alerts: a deleted or + moved search base, or one the service account may not read, stays wrong until someone + fixes it. A pass with such a refusal beside other answers is NOT this; only + ``misconfigured`` says it saw one.""" + return self.aborted == MISCONFIGURED_ABORT + @property def judged_nothing(self) -> bool: - """True when the pass aborted before judging any account: an outage, a pass of referrals - and failures only, or one of faults and failures only. Such a pass leaves the hold's state, - its message and every clear alone, because it learned nothing about the accounts it asked - about. It still strikes an UNKEYED row, which was judged without the directory (BACKLOG - #3170).""" - return self.directory_outage or self.directory_referral or self.probe_fault + """True when the pass aborted before judging any account: an outage, or a pass whose every + answer went unjudged, at least one of them referred, refused for its configuration or + faulted. Such a pass leaves the hold's state, its message and every clear alone, because + it learned nothing about the accounts it asked about. It still strikes an UNKEYED row, + which was judged without the directory (BACKLOG #3170).""" + return ( + self.directory_outage + or self.directory_referral + or self.directory_misconfigured + or self.probe_fault + ) @property def judged(self) -> int: @@ -411,8 +470,8 @@ def select_candidates( def _unkeyed_strikes(probes: list[Probe], prior_strikes: Mapping[str, int]) -> dict[str, int]: """The strikes of a pass that judged nothing (BACKLOG #3170 item 3). - An UNKEYED row was judged without the directory, so an outage or a pass of referrals or faults - only (BACKLOG #3100) still strikes it, as any other pass does. Only the strike: its outcome is + An UNKEYED row was judged without the directory, so a pass that judged nothing it asked about + (:attr:`ReconcilePlan.judged_nothing`) still strikes it, as any other pass does. Only the strike: its outcome is not recorded, so the hold's record, and so its carry-over, stays as it was, and an aborted pass revokes nothing.""" return { @@ -454,16 +513,25 @@ def plan_pass( unavailable = [p for p in probes if p.outcome is ProbeOutcome.UNAVAILABLE] referred = [p for p in probes if p.outcome is ProbeOutcome.REFERRED] faulted = [p for p in probes if p.outcome is ProbeOutcome.FAULT] + misconfigured = [p for p in probes if p.outcome is ProbeOutcome.MISCONFIGURED] # BACKLOG #2434. The caller reads a row it may not ask about as UNKEYED, so that outcome is # the one mark of "never asked": it says nothing about whether the directory answered. asked = sum(1 for p in probes if p.outcome is not ProbeOutcome.UNKEYED) - if (referred or faulted) and len(unavailable) + len(referred) + len(faulted) == asked: - # BACKLOG #2538, #3100. Nothing the pass asked was answered, as on an outage, so it judges - # nothing and the hold's state carries over. Checked before the outage, so a pass of - # referrals or faults and failures is not read as a plain outage, which pages nobody. A - # user search base in another domain of the forest refers every probe, and an entry or a - # defect that raises does so on every pass. A referral wins over a fault: it names the fix. + alerting = len(referred) + len(misconfigured) + len(faulted) + if alerting and len(unavailable) + alerting == asked: + # BACKLOG #2538, #3010, #3100. Nothing the pass asked was answered, as on an outage, so it + # judges nothing and the hold's state carries over. Checked before the outage, so such a + # pass is not read as a plain outage, which pages nobody. A user search base in another + # domain of the forest refers every probe, a deleted one is refused for every probe, and an + # entry or a defect that raises does so on every pass. A referral wins, then a + # configuration refusal: each names the fix, and a fault does not. + if referred: + aborted = REFERRAL_ABORT + elif misconfigured: + aborted = MISCONFIGURED_ABORT + else: + aborted = PROBE_FAULT_ABORT return ReconcilePlan( strikes=_unkeyed_strikes(probes, prior_strikes), probed=len(probes), @@ -471,7 +539,8 @@ def plan_pass( unavailable=len(unavailable), referred=tuple(p.user_id for p in referred), faulted=tuple(p.user_id for p in faulted), - aborted=REFERRAL_ABORT if referred else PROBE_FAULT_ABORT, + misconfigured=tuple(p.user_id for p in misconfigured), + aborted=aborted, latched=latched, ) @@ -493,7 +562,8 @@ def plan_pass( # ADR 0195 rule items 4 to 6. An UNAVAILABLE probe leaves the prior entry in place, so a # directory blip on one held account does not make another look single. A REFERRED probe does # too (BACKLOG #2538): a referral says where to look, not what the account is. So does a FAULT - # probe (BACKLOG #3100): the engine could not read the answer. Every other probe + # probe (BACKLOG #3100): the engine could not read the answer. So does a MISCONFIGURED probe + # (BACKLOG #3010): the directory refused the search, not the account. Every other probe # is judged below as on any pass. A referring group search base refers only the accounts the # user search FOUND enabled, so a disabled or absent account's answer still strikes and revokes. outcomes = {p.user_id: p.outcome for p in probes if p.outcome not in _UNJUDGED} @@ -579,6 +649,7 @@ def plan_pass( unavailable=len(unavailable), referred=tuple(p.user_id for p in referred), faulted=tuple(p.user_id for p in faulted), + misconfigured=tuple(p.user_id for p in misconfigured), outcomes=outcomes, hold=hold, latched=hold_latches(undetermined=undetermined, latched=latched), diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 807121b5e..6c0f86b47 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -195,6 +195,15 @@ {"mfa_enroll_confirm", "mfa_verify", "webauthn_enroll", "webauthn_assert"} ) +#: Vault BACKLOG #3028: what :attr:`AuthService.supersession_lineage_seconds` adds to the flow TTL +#: for the sign-in's own work after its last wait. At the federated callback that is the code +#: exchange and a JWKS refresh, each under a 10-second socket timeout, then the directory resolve, +#: whose LDAP calls each sit under ``ad_connect_timeout`` and ``ad_receive_timeout`` (10 seconds +#: by default), then the store writes before the mint. The password and directory legs are one +#: request of the same parts. A socket timeout bounds each blocking call, not the whole, so a +#: sign-in that runs past the window is possible, and docs/SECURITY.md states it as a limit. +_LINEAGE_SIGN_IN_SLACK_SECONDS: Final = 120.0 + #: What ``AuthService._apply_reconcile_revocation`` did with one planned revocation (BACKLOG #3085). _RevocationApplied = Literal["skipped", "quiet", "reported"] @@ -360,6 +369,7 @@ def _error_if_bundled_corpus_unusable(check_breached: bool) -> None: { reconcile.ProbeOutcome.UNAVAILABLE.value, reconcile.ProbeOutcome.REFERRED.value, + reconcile.ProbeOutcome.MISCONFIGURED.value, CERT_PROBE_SATURATED, } ) @@ -2079,7 +2089,8 @@ class ProvisionedAdministrator: #: The account had no notification address before the repair, so there was nobody to tell. HOLDER_NOTICE_NO_PRIOR_ADDRESS = "no_prior_address" #: The account had an address, but nothing was handed off: no channel was wired, notices are turned -#: off, or the notifier raised (each logged where it applies). +#: off, the notifier raised, or it dropped the notice on a full queue (each logged where it applies; +#: vault BACKLOG #3120 for the last). HOLDER_NOTICE_NO_CHANNEL = "no_channel" @@ -2345,12 +2356,21 @@ def _floor_exempt_marker(floor_exempt_amr: Sequence[str]) -> dict[str, object]: #: it throttles over. See :meth:`AuthService._lock_notice_due`. _LOCK_NOTICE_ACTION: Final = LOCK_NOTICE_ACTION _LOCK_NOTICE_WINDOW_SECONDS: Final = 24 * 3600.0 -#: BACKLOG #2216: how long the API lifespan's :meth:`AuthService.drain_background` lets the +#: BACKLOG #2216: how long :meth:`AuthService.close_background` lets the #: service's background tasks finish at shutdown before it cancels the rest, and how long it then -#: waits for the cancelled ones to unwind. Both are shares of the service's graceful-stop window, -#: whose budget is stated at ``DRAIN_TIMEOUT_SECONDS`` in ``api/approvals.py``; keep the sum inside -#: it. A task still running at the first bound is queued behind other throttle reads, or stuck on -#: the store. A task still unwinding at the second is left to finish or fail on its own. +#: waits for the cancelled ones to unwind. A task still running at the first bound is queued behind +#: other throttle reads, or stuck on the store. A task still unwinding at the second is left to +#: finish or fail on its own. +#: +#: The budget is NSSM's 15 s graceful-stop window (``AppStopMethodConsole`` in +#: ``scripts/service/install-service.ps1``), not any one drain's constant (vault BACKLOG #3120). The +#: API lifespans spend it through one shutdown budget. ``_SHUTDOWN_BUDGET_SECONDS`` in +#: ``api/app.py`` states the budget and the window arithmetic (BACKLOG #3191); ``_app_drain_steps`` +#: and the managed lifespan's teardown hold the steps, their order and each step's floor, from which +#: ``_run_teardown`` computes the step's bound. Where the lock-notice step's bound is below +#: :data:`_BACKGROUND_DRAIN_SECONDS`, the teardown cuts this drain before it cancels anything; +#: ``docs/SECURITY.md``, *Residual: the background task adds one way to lose a notice*, says what +#: that costs. A change to either constant is read against that budget. _BACKGROUND_DRAIN_SECONDS: Final = 2.0 _BACKGROUND_CANCEL_GRACE_SECONDS: Final = 0.5 @@ -2921,15 +2941,19 @@ def __init__( # one kind landing together read the throttle one after the other and mail once. Its own table: # the credential queue above is held across a sign-in's pad, and the notice runs off it. Per # API process, like that queue: engine shards serving their own API ports each keep one, so - # two locks landing on two of them at once can still mail twice. + # two locks landing on two of them at once can still mail twice. Declined in vault BACKLOG + # #3120, with the residual in docs/SECURITY.md. self._lock_notice_locks: dict[str, _KeyedLock] = {} # BACKLOG #2216: strong references to the service's background tasks, so a running one is not - # garbage-collected. Each removes itself when done; drain_background() awaits the rest. + # garbage-collected. Each removes itself when done; drain_background() awaits the rest. No + # cap on how many wait, declined in vault BACKLOG #3120: a cap would have to drop a lock + # notice or run it on the request path. docs/SECURITY.md states the residual. self._background_tasks: set[asyncio.Task[None]] = set() # Set by drain_background(close=True) at shutdown. After it, a lock notice runs inline, # since a task started then would outlive the drain and meet a closed store. self._background_closed = False - # One throttle read at a time (BACKLOG #2216); see _lock_notice_held_back. + # One throttle read at a time in this process, across every account (BACKLOG #2216); see + # _lock_notice_held_back. self._lock_notice_read_slots = asyncio.Semaphore(1) # Per-SESSION failed re-proofs (BACKLOG #1138): token_hash -> (failures charged to that # session, monotonic time of the first, the account's user id). At lockout_threshold the session is revoked, so a stolen session gets that many @@ -2977,6 +3001,10 @@ def __init__( #: Latched probe fault (BACKLOG #3100): set on a pass in which a probe faulted, and released #: by the next pass that judged something with no fault (`_record_reconcile_fault`). self._reconcile_fault_alert: str | None = None + #: Latched configuration refusal (BACKLOG #3010), on the probe fault's rule: set on a pass in + #: which the directory refused a probe for its configuration, and released by the next pass + #: that judged something with no such refusal (`_record_reconcile_misconfigured`). + self._reconcile_misconfigured_alert: str | None = None #: user_ids a breaker trip has not yet seen read PRESENT on a pass that was not aborted #: (BACKLOG #2136). `_mark_reconcile_clears` reports no breaker clear while one is signed in. self._reconcile_unconfirmed: set[str] = set() @@ -4846,6 +4874,21 @@ def oidc_flow_ttl_seconds(self) -> int: """The staged-flow lifetime, so the browser cookie's max-age matches the server-side TTL.""" return self._settings.oidc_flow_ttl_seconds + @property + def supersession_lineage_seconds(self) -> float: + """How long a retired session hash still leads a sign-in's supersession to the session it + became (vault BACKLOG #3028, ADR 0210), in seconds. The session reaper purges links older + than this. + + It must outlast the longest sign-in a rotation can race, because the presented hash is read + at the START of the sign-in and the supersession runs at its end. The longest is the + federated leg: its start leg stages the hash, and the callback can come back as late as + ``oidc_flow_ttl_seconds`` after. :data:`_LINEAGE_SIGN_IN_SLACK_SECONDS` covers the + callback's own work after that, and every other leg is one request inside it. A longer + window only lengthens the time a stale copy of a token can END its successor; it never + lets one authenticate.""" + return float(self._settings.oidc_flow_ttl_seconds) + _LINEAGE_SIGN_IN_SLACK_SECONDS + def _oidc_redirect_uri(self, public_origin: str) -> str: return public_origin.rstrip("/") + self._settings.oidc_redirect_path @@ -6851,6 +6894,13 @@ def directory_reconcile_fault(self) -> str | None: pass judges something with no fault (:meth:`_record_reconcile_fault`).""" return self._reconcile_fault_alert + @property + def directory_reconcile_misconfigured(self) -> str | None: + """The standing configuration refusal's operator message, or ``None`` (BACKLOG #3010). + Latches until a pass judges something with no such refusal + (:meth:`_record_reconcile_misconfigured`).""" + return self._reconcile_misconfigured_alert + @property def directory_reconcile_hold(self) -> str | None: """The engaged undetermined-wave hold's operator message, or ``None`` (ADR 0195). Latches @@ -6922,13 +6972,27 @@ async def _probe_principal(self, user: UserRecord) -> reconcile.Probe: # the arm above reads as REFERRED (vault BACKLOG #3697 step 2). return _fault_probe(user, exc) except LdapError as exc: + if isinstance(exc, LdapUnansweredError) and exc.configuration: + # BACKLOG #3010. A deleted or moved search base, or one the service account may not + # read, refuses every probe on every pass. Read as UNAVAILABLE it would stop + # revocation without a page; MISCONFIGURED alerts instead, and never strikes or + # revokes this account. DEBUG here, as for a referral: the reconciler logs one + # ERROR per pass carrying the first refusal's text. That text is the engine's own, + # naming the operation and the code, never the directory's diagnostic message. + _log.debug("directory probe of %s was refused: %s", user.username, exc) + return reconcile.Probe( + user.id, user.username, reconcile.ProbeOutcome.MISCONFIGURED, detail=str(exc) + ) # FAIL OPEN, for the reconciler. verify_mfa reads the same UNAVAILABLE as a refusal and # fails closed, because it grants rather than revokes (BACKLOG #2023). It writes an audit # row per refusal, so it adds no log line here either. # An unreachable DC must never revoke: a fail-closed re-check would turn a # directory blip into a total console outage during exactly the incident when operators # need the console. Debug-level — a flapping DC must not flood the log at one line per - # user per pass; the pass-level summary below reports the count at WARNING. + # user per pass; the pass-level summary below reports the count at WARNING. A result + # code that is not an answer is also reported at WARNING by ldap.py itself, once per + # operation and code (BACKLOG #3010), in engine text that never quotes the directory's + # diagnostic message. An undecodable response does not reach here (the arm above). _log.debug("directory probe failed for %s: %s", user.username, exc) # Vault BACKLOG #3696. A group search past its deadline or its entry cap is still # unjudged, so still UNAVAILABLE, but it says which bound refused it, so it pages. @@ -7018,8 +7082,9 @@ def _note_paging_refusals( stops is the role and scope re-diff of an enabled account. Unlike an outage it recurs on every pass, so it pages, once per state change: an account already latched is not returned again, whichever bound refused it this time. An account the pass judged - (``judged``, the plan's ``outcomes``) re-arms here; an outage, a referral or a fault judged - nothing, says nothing about the bound, and leaves the latch alone. **The latch is set only once the entry + (``judged``, the plan's ``outcomes``) re-arms here; an outage, a referral, a configuration + refusal (BACKLOG #3010) or a fault judged nothing, says nothing about the bound, and leaves + the latch alone. **The latch is set only once the entry is recorded** (:meth:`_record_paging_refusals`), so a pass that raises in the service before that point leaves the account to enter again on the next pass. A sink that raises in the lifespan task after it can still cost the page; the row stands. An account that @@ -7033,7 +7098,7 @@ def _note_paging_refusals( for probe in probes: if ( probe.outcome is reconcile.ProbeOutcome.UNAVAILABLE - and probe.detail is not None # REFERRED and FAULT carry a detail of their own + and probe.detail is not None # REFERRED, FAULT, MISCONFIGURED carry their own and probe.user_id not in latch ): entered.append((probe.user_id, probe.username, probe.detail)) @@ -7071,7 +7136,8 @@ async def reconcile_directory_sessions(self) -> reconcile.ReconcilePlan: * a probe that could not reach the directory contributes nothing (fail-open); * a probe the directory referred contributes nothing either, and the pass alerts, while - every other account is still judged (BACKLOG #2538); + every other account is still judged (BACKLOG #2538). So does a probe the directory + refused for its configuration, such as a deleted search base (BACKLOG #3010); * a principal must come back absent, disabled, undetermined or unkeyed ``ad_session_recheck_strikes`` passes running; * a PRESENT principal whose directory groups would change its roles, or withdraw or narrow @@ -7276,20 +7342,25 @@ async def reconcile_directory_sessions(self) -> reconcile.ReconcilePlan: # anything below can raise or return, so every pass that judged or saw a condition counts. self._track_reconcile_episodes(plan, users, announced=announced) referral = next((p for p in probes if p.outcome is reconcile.ProbeOutcome.REFERRED), None) + refusal = next( + (p for p in probes if p.outcome is reconcile.ProbeOutcome.MISCONFIGURED), None + ) if plan.aborted is not None: - if not plan.directory_referral and not plan.probe_fault: + if plan.directory_outage or not plan.judged_nothing: + # The outage's skipped row, or the breaker's trip. A pass that judged nothing for + # any other reason is recorded only by the recorders below, as a referral (#2538), + # a configuration refusal (#3010) or a fault (#3100), and never as a trip. await self._abort_reconcile_pass(plan) if not plan.directory_outage: - # A pass of referrals only is recorded as a referral and nothing else (#2538), and - # a pass of faults only as a fault and nothing else (#3100). await self._record_reconcile_referral(plan, first=referral) + await self._record_reconcile_misconfigured(plan, first=refusal) await self._record_reconcile_fault(plan) # Vault BACKLOG #3696. On every abort, an outage-shaped one included, as its page is. await self._record_paging_refusals(plan) if not plan.judged_nothing: # A held pass writes its own row even when the breaker also aborts it (ADR 0195 rule - # item 9). An outage or a pass of referrals only judged nothing, so it leaves the - # hold's message alone and marks no clear (BACKLOG #2538). + # item 9). A pass that judged nothing (an outage, or one whose every answer went + # unjudged) leaves the hold's message alone and marks no clear (BACKLOG #2538). await self._record_reconcile_hold(plan) plan = self._mark_reconcile_clears(plan, users) await self._report_unkeyed_bindings(unkeyed, still_unkeyed=still_unkeyed) @@ -7350,7 +7421,8 @@ async def reconcile_directory_sessions(self) -> reconcile.ReconcilePlan: # BACKLOG #2538. After the revocations, like the hold below: the referred accounts were left # unjudged, and every other account's revocation above has already been applied. await self._record_reconcile_referral(plan, first=referral) - # BACKLOG #3100. After the revocations too, for the same reason as the referral's. + # BACKLOG #3010 and #3100. After the revocations too, for the same reason as the referral's. + await self._record_reconcile_misconfigured(plan, first=refusal) await self._record_reconcile_fault(plan) # Vault BACKLOG #3696. After the revocations too, for the same reason. await self._record_paging_refusals(plan) @@ -7409,11 +7481,12 @@ def _mark_reconcile_clears( referred account has no answer from this pass, and an older one on record may predate the referral. So while a referred account is probed, a standing referral keeps the breaker's and the hold's instances open after their own causes are gone: a missed - clear, never a false one (BACKLOG #3100 item 7). A pass with a probe fault marks none - either, for the same reason. For these two clears neither is carried to later passes: - a later pass whose sample misses the referred or faulting account reads its older - answer on record and can still mark one (left open under BACKLOG #3100). Only the - referral's own clear below carries its accounts across passes. + clear, never a false one (BACKLOG #3100 item 7). A pass with a probe fault, or with a + probe refused for its configuration (BACKLOG #3010), marks none either, for the same + reason. For these two clears none is carried to later passes: a later pass whose + sample misses the referred, faulting or refused account reads its older answer on + record and can still mark one (left open under BACKLOG #3100). Only the referral's own + clear below carries its accounts across passes. * The referral's own instance is clear when, on top of the first test, every account signed in at the last referral has since read PRESENT on a pass with no referral (``_reconcile_referred``), and ``_reconcile_referral_standing`` is ``"referred"``: a @@ -7474,11 +7547,18 @@ def _mark_reconcile_clears( revoked_undetermined = undetermined in (self._reconcile_outcomes.get(u) for u in revoked) ids = [uid for uid in users if uid not in revoked] outcomes = [self._reconcile_outcomes.get(uid) for uid in ids] - # A faulted account, like a referred one, keeps an older answer that may predate the fault - # (BACKLOG #3100), so a pass with either marks no clear. - covered = bool(ids) and None not in outcomes and not plan.referred and not plan.faulted + # A faulted or refused account, like a referred one, keeps an older answer that may predate + # the fault or the refusal (BACKLOG #3100, #3010), so a pass with any of them marks no clear. + covered = ( + bool(ids) + and None not in outcomes + and not plan.referred + and not plan.faulted + and not plan.misconfigured + ) settled = covered and undetermined not in outcomes - # Called only on a pass that judged something, so a pass of referrals only never gets here. + # Called only on a pass that judged something, so a pass that judged nothing never gets + # here. referral_clear = ( covered and self._reconcile_referral_standing == "referred" @@ -7600,7 +7680,7 @@ def _track_reconcile_episodes( counts, a held one too, so an account that never reads PRESENT does not keep a trip or a referral open for good. For a hold the answer must not be undetermined: a lone account that is held, then strikes, then is held again is one hold, not one per flap. A probe - that reached no answer (unavailable, referred, faulted), a probe sample that misses the + that reached no answer (unavailable, referred, faulted, misconfigured), a probe sample that misses the accounts, and a pass that judged nothing end nothing, so one condition seen across a rotation or an outage stays one episode. @@ -8232,8 +8312,8 @@ async def _abort_reconcile_pass(self, plan: reconcile.ReconcilePlan) -> None: ) return # The ceiling is taken over the breaker's own denominator, which counts held probes (ADR - # 0195 amendment 2026-10-11, BACKLOG #2138). A pass of referrals only never reaches here: it - # is not a trip (BACKLOG #2538). + # 0195 amendment 2026-10-11, BACKLOG #2138). No other pass that judged nothing reaches + # here: none of them is a trip (BACKLOG #2538, #3010, #3100). ceiling = reconcile.breaker_ceiling( probed=plan.judged, max_absolute=self._settings.ad_session_revoke_max, @@ -8266,8 +8346,9 @@ async def _record_reconcile_referral( ) -> None: """Latch, log and audit a pass in which the directory referred probes. - BACKLOG #2538. Called on every pass that judged something, and on a pass of referrals - only; an outage pass does not call it. On one with a referral, + BACKLOG #2538. Called on every pass but an outage, the same as + :meth:`_record_reconcile_misconfigured` and :meth:`_record_reconcile_fault`. On one with a + referral, whatever else the pass did, it latches the message, logs it at ERROR with the first referral's search and hosts, and writes an ``auth.ad_reconcile_referred`` row. Its own audit action, because a pass with a referral beside other answers is not an aborted one, and an @@ -8291,18 +8372,10 @@ async def _record_reconcile_referral( if plan.aborted is not None else "Accounts that answered without a referral were judged as usual." ) - if plan.faulted: - # BACKLOG #3100. Counted here too, and recorded as a fault of their own. - others = f"{len(plan.faulted)} raised a probe fault. {others}" - if plan.unavailable: - # A pass of referrals and failures is recorded here and not as an outage, so the - # failures are counted here, or a near-total outage would read as a search-base fault. - others = ( - f"{plan.unavailable} could not be read (the directory was unreachable). {others}" - ) + others = self._unjudged_others(plan, others, faults=True, refusals=True) self._reconcile_referral_alert = ( - "LDAP referral: check [auth].ad_user_search_base and, for nested groups, " - "[auth].ad_group_search_base; one names a base in another domain of the forest. " + "LDAP referral: check [auth].ad_user_search_base, [auth].ad_group_search_base for " + "nested groups, and [auth].ad_bind_dn; one names another domain of the forest. " f"{len(plan.referred)} of {plan.asked} signed-in directory account(s) probed were " f"referred and left unjudged. {others} A referred account is never revoked while it " "is referred, because the engine does not follow referrals. Use a base in the bound " @@ -8347,10 +8420,12 @@ async def _record_reconcile_fault(self, plan: reconcile.ReconcilePlan) -> None: source for the reason ``api/app.py::_alert_reconcile_plan`` states. **Nothing resolves that instance automatically**: an operator resolves it once the fault is gone. - Called on every pass that judged something, and on a pass of referrals or faults only; an - outage pass does not call it. A pass with no fault that judged something releases the - latched message. The message and the row carry counts and no username; the per-probe - WARNING in :meth:`_probe_principal` names each account and the exception type. + Called on every pass but an outage. Whatever else a pass did, one with no fault that judged + something releases the latched message, and one that judged nothing leaves it as it was. + The message, the row and the alert count the faults over the probes ASKED, as the + referral's do (BACKLOG #3170): an UNKEYED row was never sent, so it cannot fault. The + message and the row carry counts and no username; the per-probe WARNING in + :meth:`_probe_principal` names each account and the exception type. """ if not plan.faulted: if not plan.judged_nothing: @@ -8361,12 +8436,7 @@ async def _record_reconcile_fault(self, plan: reconcile.ReconcilePlan) -> None: if plan.aborted is not None else "Accounts that answered were judged as usual." ) - if plan.unavailable: - # Counted here, as the referral's message does: a pass of faults and failures is - # recorded as a fault and not as an outage, so the outage would otherwise go unsaid. - others = ( - f"{plan.unavailable} could not be read (the directory was unreachable). {others}" - ) + others = self._unjudged_others(plan, others, refusals=True) # The pointer to the account comes first: the store keeps only the first 200 characters # of an alert instance's reason, as `_record_reconcile_referral` says. self._reconcile_fault_alert = ( @@ -8384,12 +8454,98 @@ async def _record_reconcile_fault(self, plan: reconcile.ReconcilePlan) -> None: "reason": reconcile.PROBE_FAULT_ABORT, "faulted": len(plan.faulted), "unavailable": plan.unavailable, - "probed": plan.probed, + "asked": plan.asked, "aborted": plan.aborted, } ), ) + async def _record_reconcile_misconfigured( + self, plan: reconcile.ReconcilePlan, *, first: reconcile.Probe | None + ) -> None: + """Latch, log and audit a pass in which the directory refused a probe for its configuration. + + BACKLOG #3010. A search that ends with noSuchObject, invalidDNSyntax or + insufficientAccessRights says the configured base, or the service account's right to read + it, is wrong, not that the account is gone (:attr:`~reconcile.ProbeOutcome.MISCONFIGURED`). + It leaves that account unjudged: no strike and no revocation. A deleted or moved search base + refuses every account on every pass, so while it lasts nothing is revoked and the sessions + run to the absolute cap. Before #3010 such a search read as "not found", which tripped the + mass-revoke breaker on a large enough estate, and a plain outage would page nobody. So + every pass with such a refusal latches the message, logs it at ERROR with the first + refusal's text, and writes an ``auth.ad_reconcile_misconfigured`` row, and the lifespan + task raises it as ``ad_reconcile_aborted`` with reason ``directory_misconfigured`` under its + own source label, for the reasons :meth:`_record_reconcile_fault` gives. Like the fault's, + **nothing resolves that instance automatically**, and the latched message follows the + fault's rule. + + Called on every pass but an outage. The message, the row and the alert count over the + probes ASKED (BACKLOG #3170). They carry counts and no username; the refusal's text names + the operation and the code, never the directory's diagnostic message, and + :meth:`_probe_principal` names each refused account at DEBUG. + """ + if not plan.misconfigured: + if not plan.judged_nothing: + self._reconcile_misconfigured_alert = None + return + others = ( + "The pass revoked nothing." + if plan.aborted is not None + else "Accounts that answered were judged as usual." + ) + others = self._unjudged_others(plan, others, faults=True) + # The settings to change come first: the store keeps only the first 200 characters of an + # alert instance's reason, as `_record_reconcile_referral` says. + self._reconcile_misconfigured_alert = ( + "LDAP configuration refused: check [auth].ad_user_search_base, " + "[auth].ad_group_search_base and the read rights of [auth].ad_bind_dn. " + f"{len(plan.misconfigured)} of {plan.asked} signed-in directory account(s) asked were " + f"refused for the configuration and left unjudged. {others} A refused account is " + "never revoked while the refusal lasts, so its sessions run to " + "[auth].session_absolute_hours. The ERROR log line for each such pass names the code." + ) + _log.error( + "directory reconcile: %s First refusal: %s", + self._reconcile_misconfigured_alert, + first.detail if first is not None else "?", + ) + await self._audit_reconciler_row( + "auth.ad_reconcile_misconfigured", + detail=_json( + { + "reason": reconcile.MISCONFIGURED_ABORT, + "misconfigured": len(plan.misconfigured), + "unavailable": plan.unavailable, + "asked": plan.asked, + "aborted": plan.aborted, + } + ), + ) + + @staticmethod + def _unjudged_others( + plan: reconcile.ReconcilePlan, others: str, *, faults: bool = False, refusals: bool = False + ) -> str: + """``others`` led by the counts of the pass's other unjudged probes, for a latched message. + + A pass whose every answer went unjudged is recorded by its referral, refusal or fault and + not as an outage, so its failures are counted here, or a near-total outage would read as a + configuration fault. ``faults`` and ``refusals`` add those counts too, each passed by every + recorder but its own kind's, so each message names the other unjudged kinds, which are + recorded under their own alerts as well.""" + if faults and plan.faulted: + others = f"{len(plan.faulted)} raised a probe fault. {others}" + if refusals and plan.misconfigured: + others = f"{len(plan.misconfigured)} were refused for the configuration. {others}" + if plan.unavailable: + # UNAVAILABLE also carries an account whose group search the engine refused for its own + # paging bound (vault BACKLOG #3696), which pages on its own, so this names both. + others = ( + f"{plan.unavailable} could not be read (the directory was unreachable, or the " + f"engine refused a group search for its own bound). {others}" + ) + return others + async def _audit_reconciler_row( self, action: str, @@ -8401,19 +8557,19 @@ async def _audit_reconciler_row( """Write one of the reconcile pass's rows, and say whether it was written. BACKLOG #2137. These rows are the held row, the breaker's aborted row, the outage's skipped - row, the referral's row (BACKLOG #2538), the probe fault's row (BACKLOG #3100) and the - unkeyed-binding report. They include each paging refusal's - ``auth.ad_reconcile_paging_refused`` row (vault BACKLOG #3696), which names the account as - ``actor``. Since BACKLOG #3018 they also include each account's + row, the referral's row (BACKLOG #2538), the probe fault's row (BACKLOG #3100), the + configuration refusal's row (BACKLOG #3010) and the unkeyed-binding report. They include + each paging refusal's ``auth.ad_reconcile_paging_refused`` row (vault BACKLOG #3696), which + names the account as ``actor``. Since BACKLOG #3018 they also include each account's ``auth.ad_session_revoked`` row and, on the reconciler's call only, a rename's ``auth.ad_username_refreshed`` and ``auth.ad_username_refresh_conflict`` rows; those name the account as ``actor``. **A failed write here does not end the pass.** If it raised, the pass would return no plan, and the lifespan task would raise no alert for the revocations - already applied, nor for the hold, the breaker, the referral or the probe fault. It would - also skip every account after this one. So a store refusal of a type - :func:`_audit_write_errors` names is logged at ERROR, naming the action and the actor, and - the pass goes on. Any other error still raises and ends the pass. The log line is the only - record of a refused row: nothing retries it. + already applied, nor for the hold, the breaker, the referral, the probe fault, the + configuration refusal or a paging refusal. It would also skip every account after this + one. So a store refusal of a type :func:`_audit_write_errors` names is logged at ERROR, + naming the action and the actor, and the pass goes on. Any other error still raises and + ends the pass. The log line is the only record of a refused row: nothing retries it. A revocation or rename these rows record has already committed; the conflict row records a rename that was refused. A revocation's notice is sent inline, so it still goes. **A @@ -9834,7 +9990,8 @@ def _note_cert_directory_answer(self, answer: reconcile.Probe | str) -> None: """Log the certificate path's directory refusals without logging one line per request. An outage is a refusal that says the directory could not be asked: unreachable, referring, - or the probe cap full. It logs one WARNING per outage and, if that + refusing the search for its configuration (``misconfigured``, BACKLOG #3010), or the probe + cap full. It logs one WARNING per outage and, if that WARNING was logged, one INFO on the next answer the directory actually gave, whatever it said about the account. WARNINGs are at least :data:`_CERT_OUTAGE_LOG_INTERVAL_SECONDS` apart, so an outage that flaps (a full cap turning over, or one referring account beside a @@ -9970,8 +10127,12 @@ async def _supersede_session_hash(self, prior_hash: str, *, client: str | None) turn every sign-in into a forced sign-out of every other device. The prior session may belong to a different user, for example on a shared workstation. It is - still ended. Anyone holding that token could already end it with ``POST /auth/logout``, so - this grants no new power. The audit row's actor is the session's OWNER, not whoever signed + still ended. Anyone holding that LIVE token could already end it with ``POST + /auth/logout``, so ending the presented session grants no new power. Following the lineage + does grant one, and it is bounded: for :attr:`supersession_lineage_seconds` after an + elevation, a holder of the RETIRED token can end the session it became through a sign-in + that succeeds, which ``logout`` cannot do. docs/SECURITY.md states it. The audit row's + actor is the session's OWNER, not whoever signed in: ``/me/security-events`` selects rows by actor, so the event belongs in the feed of the user whose session ended, and must not put that user's ids in anyone else's. @@ -9981,18 +10142,27 @@ async def _supersede_session_hash(self, prior_hash: str, *, client: str | None) session that was still LIVE gets an audit row, so the trail does not record the ending of something that had already ended. - **The read and the revoke are one atomic store operation (BACKLOG #2146).** Why that closes - the race with a concurrent rotation is :meth:`AuthStore.supersede_session`'s to say. It holds + **The read and the revoke are one store transaction (BACKLOG #2146).** Why that closes the + race with a concurrent rotation is :meth:`AuthStore.supersede_session`'s to say. It holds across engine processes too, because engine shards serve the API from one shared store. - **A rotation that committed BEFORE that operation is a stated limit, not a race this - closes.** The presented hash then names no session, so this ends nothing, and the session - lives on under its new token. docs/SECURITY.md (ASVS 7.2.4, the supersession paragraphs) - states how wide that window is. + **A rotation that committed BEFORE that transaction is followed (vault BACKLOG #3028, ADR + 0210).** An elevation in another tab can finish while this sign-in is in flight, and on the + federated leg the presented hash was read at the start leg, before the identity provider's + round trip. The presented hash then names no session. The store follows the lineage links + each rotation wrote to the session's live end and revokes that, in the same transaction. + Only links retired within :attr:`supersession_lineage_seconds` of now are followed, so this + is the one place a retired hash leads anywhere, and it leads only to an end. An elevation + that finished longer ago than that, such as one a second holder of the token completed + earlier, is not followed; docs/SECURITY.md (ASVS 7.2.4, the supersession paragraphs) states + that bound. The audit row names the session that ended and, when a link was followed, the + hash that was presented. Returns True when it audited. """ before = time.time() - prior = await self._store.supersede_session(prior_hash, now=before) + prior = await self._store.supersede_session( + prior_hash, now=before, lineage_since=before - self.supersession_lineage_seconds + ) if prior is None: return False after = time.time() @@ -10006,10 +10176,13 @@ async def _supersede_session_hash(self, prior_hash: str, *, client: str | None) if not prior.is_live(now=moment, idle_seconds=self.session_idle_seconds): return False owner = await self._load_user(prior.user_id) + detail = {"scope": "superseded", "session": prior.token_hash[:12]} + if not constant_time_equal(prior.token_hash, prior_hash): # ASVS 11.2.4, as elsewhere + detail["presented"] = prior_hash[:12] await self._audit( "auth.session_revoked", actor=owner.username if owner is not None else None, - detail=_json({"scope": "superseded", "session": prior_hash[:12]}), + detail=_json(detail), client=client, ) return True @@ -11088,9 +11261,9 @@ async def _classify_login_address(self, user: UserRecord, client: str | None) -> **THE BASELINE IS THE ACCOUNT'S KNOWN-ADDRESS RECORD** (vault BACKLOG #2145): the host keys that :meth:`_mark_login_address_known` writes, which says when, read through the record's - primary key on the account id. It replaced a baseline read from ``audit_log``, which has no - actor index and could not tell a finished sign-in from one that stopped at the password on - the directory leg. Keyed on the + primary key on the account id. It replaced a baseline read from ``audit_log``, which then had + no actor index and could not tell a finished sign-in from one that stopped at the password + on the directory leg. Keyed on the account id, so a re-created namesake inherits nothing, and the account's deletion removes it. Only rows seen within ``_LOGIN_ADDRESS_LOOKBACK_SECONDS`` count. Addresses compare by @@ -15182,10 +15355,12 @@ async def _lock_notice_held_back(self, user: UserRecord, lock: str) -> bool: """The throttle's READ: whether a notice of this ``lock`` kind already went out in the window. Kept apart from the row write, so the background task can send the mail between this read - and :meth:`_record_lock_notice` (BACKLOG #2216). One such read runs at a time per - service (``_lock_notice_read_slots``). It walks every recent ``auth.lock_notice`` row of - every account, since the audit log has no actor index. Many locks at once would otherwise - fill the store's read pool, which sign-ins wait on too. + and :meth:`_record_lock_notice` (BACKLOG #2216). An audit index serves it (vault BACKLOG + #3120); ``docs/SECURITY.md`` names it and says on which backends its use is measured. One + such read still runs at a time per service (``_lock_notice_read_slots``), so locks on many + accounts at once cannot fill the store's read pool, which sign-ins wait on too. The cost is + that each notice waits behind the reads queued before it, across every account in this + process; ``docs/SECURITY.md`` states it. A failed read fails OPEN (``False``) and is logged by exception class alone. The line names neither the account nor the notice kind (BACKLOG #1131): it runs only when a lock lands, @@ -15200,7 +15375,8 @@ async def _lock_notice_held_back(self, user: UserRecord, lock: str) -> bool: ) except Exception as exc: # noqa: BLE001 - fails open by design, and is logged _log.error( - "a security-notice throttle read failed (%s); the notice was sent unthrottled", + "a security-notice throttle read failed (%s); the throttle did not hold the notice " + "back, and whether it is mailed is up to the notifier", type(exc).__name__, ) return False @@ -15219,9 +15395,16 @@ async def _record_lock_notice(self, user: UserRecord, lock: str, *, handed: bool """Write the ``auth.lock_notice`` row the throttle reads, after the mail. ``mailed`` is true only when the account had an address AND the notifier took the event - (``handed``, BACKLOG #2019). A refused hand-off then writes ``mailed: false``, which holds - back no later mailable notice, so the next lock mails again rather than staying quiet for - a day.""" + (``handed``, BACKLOG #2019). A refused hand-off, or one the notifier dropped on a full + queue or refused once it had begun to close at shutdown (vault BACKLOG #3120), then writes + ``mailed: false``, which holds back no later + mailable notice, so the next lock mails again rather than staying quiet for a day. + + **A send that fails after the hand-off still leaves ``mailed: true``.** It fails on the + notifier's own task, after this row is written, and is recorded only on the notifier's + ``auth.lock_notice_undelivered`` row (BACKLOG #2383), which the throttle does not read. So + the next lock of the kind is held back for the rest of the window. A residual + ``docs/SECURITY.md`` states.""" await self._audit( _LOCK_NOTICE_ACTION, actor=user.username, @@ -15242,9 +15425,11 @@ async def _notify_security( or an admin action. The caller writes the audit row, not this method: see :meth:`_record_suspicious_login` for the two 6.3.5 events. - Returns ``True`` only when the notifier took the event without raising (BACKLOG #2019), so a - caller that audits the outcome can record what happened. That is a hand-off, not a delivery. - Most callers ignore it. + Returns ``True`` only when the notifier took the event without raising (BACKLOG #2019) and + said it took it: the shipped notifier answers ``False`` for an event it dropped, for the + reasons the ``SecurityNotifier`` contract lists (vault BACKLOG #3120). So a caller that audits the outcome + can record what happened. That is a hand-off, not a delivery: a send that fails later, on + the notifier's own task, is not reported back. Most callers ignore it. The docstring used to promise both arms were logged while only the failure arm was (BACKLOG #1139), so read the branches rather than this paragraph if they ever diverge again.""" @@ -15291,7 +15476,7 @@ async def _notify_security( ) return False try: - await self._security_notifier.notify( + taken = await self._security_notifier.notify( SecurityEvent( event_type=event_type, username=username, @@ -15315,4 +15500,6 @@ async def _notify_security( type(exc).__name__, ) return False - return True + # bool(): a notifier outside the contract that returns nothing reads as not taken, so the + # cheap failure follows (a duplicate lock notice), never a day held back. + return bool(taken) diff --git a/messagefoundry/config/settings.py b/messagefoundry/config/settings.py index bcaecf6b1..3eef0fff1 100644 --- a/messagefoundry/config/settings.py +++ b/messagefoundry/config/settings.py @@ -3384,13 +3384,21 @@ class AuthSettings(_Section): # effective interval instead of a bind storm against the DC. ad_session_recheck_max_users: int = 200 # --- mass-revoke circuit breaker --- - # A misconfigured search base, a moved OU, or a service account that lost read rights on the - # entries returns "not found" for EVERY user — indistinguishable from "everyone was deleted". - # Without a brake the reconciler would sign out the entire estate during exactly the incident when - # operators need the console. A pass that would revoke more than BOTH of these thresholds aborts, + # A search base that names the wrong OU, an OU reorganisation that moves the users out of it, or + # a service account that lost read rights on the entries while it can still read the base returns + # "not found" for EVERY user — indistinguishable from "everyone was deleted". Without a brake + # the reconciler would sign out the entire estate during exactly the incident when operators need + # the console. A pass that would revoke more than BOTH of these thresholds aborts, # revokes nothing, and raises a loud operator-visible alert (log ERROR + an # `auth.ad_reconcile_aborted` audit row). # + # A base that was deleted or moved, or that the service account may not read at all, usually ends + # the search with a result code instead (noSuchObject, invalidDNSyntax, insufficientAccessRights): + # a configuration refusal (BACKLOG #3010). It alerts on its own `ad_reconcile_aborted` source. + # The refused account is neither struck nor revoked, but it stays in this breaker's denominator, + # the accounts probed (`ReconcilePlan.probed`). When such a pass aborts before the breaker runs + # is stated once in docs/SECURITY.md. + # # BOTH must be exceeded to trip, deliberately: the absolute floor stops the breaker firing on a tiny # estate where any proportion is meaningless (3 of 3 genuine offboardings is 100 %), and the # proportion stops a large estate being signed out wholesale. Requiring both means it fires only on diff --git a/messagefoundry/pipeline/alert_sinks.py b/messagefoundry/pipeline/alert_sinks.py index 90cddabaf..ea670535b 100644 --- a/messagefoundry/pipeline/alert_sinks.py +++ b/messagefoundry/pipeline/alert_sinks.py @@ -171,8 +171,9 @@ def __init__(self, *, max_queue: int = _MAX_QUEUE) -> None: self._queue: asyncio.Queue[_T | None] = asyncio.Queue(maxsize=max_queue) self._task: asyncio.Task[None] | None = None - def _enqueue(self, item: _T, *, dropped: str | None) -> None: + def _enqueue(self, item: _T, *, dropped: str | None) -> bool: """Non-blocking enqueue; on a full queue, drop the item with a warning (``dropped`` names it). + Returns whether the item was queued. ``dropped=None`` drops without a line, for an item whose drop must not show in the general log (a lock notice, BACKLOG #1131; see ``auth.notifications.LOG_SILENT_EVENT_TYPES``).""" @@ -181,6 +182,8 @@ def _enqueue(self, item: _T, *, dropped: str | None) -> None: except asyncio.QueueFull: if dropped is not None: log.warning("%s queue full; dropping %s", type(self).__name__, dropped) + return False + return True def start(self) -> None: if self._task is None: @@ -1178,8 +1181,8 @@ def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: # ADR 0079 mechanism 2: the mass-revoke breaker tripped and the pass applied nothing, or one - # or more probes were referred (BACKLOG #2538) or faulted (BACKLOG #3100) and left unjudged; - # `reason` tells which. The + # or more probes were referred (BACKLOG #2538), refused for the configuration (BACKLOG + # #3010) or faulted (BACKLOG #3100) and left unjudged; `reason` tells which. The # fixed source label keys the throttle, so a standing misconfiguration that trips every pass # pages once per cooldown rather than once per pass. `detail` is the latched operator explanation, # and it is what the instance's reason column shows (detail wins over reason there). diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index b247ffa40..496c93057 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -594,26 +594,29 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s which leaves the referred accounts unjudged and may sit beside revocations the pass applied; or one or more probes raised a fault the engine could not read (``probe_fault``, BACKLOG #3100, the ``auth.ad_reconcile_faulted`` row), which leaves those accounts unjudged the same - way; or one account's group search was refused for the engine's own bound + way; or the directory refused one or more probes with a result code that points at the + configuration, such as a deleted or moved search base (``directory_misconfigured``, BACKLOG + #3010, the ``auth.ad_reconcile_misconfigured`` row), which leaves those accounts unjudged the + same way; or one account's group search was refused for the engine's own bound (``group_search_deadline`` or ``group_search_cap``, vault BACKLOG #3696), raised once as - the account enters that state, which leaves it unjudged too. The type's name predates all - but the first case. Read ``reason`` before naming the cause. + the account enters that state, which leaves it unjudged too. The type's name predates every + case but the first. Read ``reason`` before naming the cause. ``name`` labels the source: ``"directory-reconciler"`` for the breaker, - ``"directory-reconciler-referral"`` for a referral, - ``"directory-reconciler-probe-fault"`` for a probe fault, and - ``"directory-reconciler-paging:"`` for a paging refusal, so each is its own - instance and throttle. ``reason`` is one of those five closed-set slugs. ``probed`` is, for - the breaker, how many principals the pass judged, its denominator; for a probe fault and a - paging refusal, the same total of candidates the pass took up, asked or not, which counts - any id-less row and, for a probe fault, the faulted accounts it left unjudged; and for a - referral how many probes the pass sent to the directory, which leaves out an id-less row - (BACKLOG #3170). A paging refusal concerns one account, and its count is still the pass's. - ``detail`` is the operator-facing explanation the auth service latches, or for a paging - refusal a fixed sentence naming its cause. No PHI. Emitted by the - API-lifespan reconciler task, never from ``auth/``. A whole-directory outage is NOT this - event: it is - audited as ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine.""" + ``"directory-reconciler-referral"`` for a referral, ``"directory-reconciler-probe-fault"`` + for a probe fault, ``"directory-reconciler-misconfigured"`` for a configuration refusal, + and ``"directory-reconciler-paging:"`` for a paging refusal, so each is its own + instance and throttle. ``reason`` is one of those six closed-set slugs. ``probed`` is, for + the breaker, how many principals the pass judged, its denominator, and a paging refusal + carries that same count of candidates the pass took up, asked or not, which counts any + id-less row: a paging refusal concerns one account, and its count is still the pass's. For + a referral, a configuration refusal and a probe fault it is how many probes the pass sent to + the directory, which leaves out an id-less row (BACKLOG #3170). ``detail`` is the + operator-facing explanation the auth service latches, or for a paging refusal a fixed + sentence naming its cause. No PHI. Emitted by the API-lifespan reconciler task, never from + ``auth/``. A whole-directory outage is NOT this event: it is audited as + ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine. A pass where + every probe fails and none is referred, refused or faults is such an outage.""" ... def ad_reconcile_held(self, name: str, *, reason: str, undetermined: int, detail: str) -> None: @@ -1178,9 +1181,9 @@ def no_enabled_administrator(self, name: str, *, via: str) -> None: def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: log.warning( - # The count is the candidates the pass took up for the breaker, a probe fault and a - # paging refusal, or a referral's probes asked (BACKLOG #3170); see - # AlertSink.ad_reconcile_aborted. + # The count is the candidates the pass took up for the breaker and a paging refusal, or + # the probes the pass asked for a referral, a configuration refusal and a probe fault + # (BACKLOG #3170); see AlertSink.ad_reconcile_aborted. "ALERT ad_reconcile_aborted: %r left accounts unrevoked or unjudged, in a pass counting " "%d principal(s) " "(%s): %s", diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 5fb39f852..06d28164f 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -541,7 +541,10 @@ def _build_body(event: SecurityEvent) -> str: class SecurityEventNotifier(_BackgroundDispatcher[SecurityEvent]): - """Emails the affected user about a security event, on a bounded background queue.""" + """Emails the affected user about a security event, on a bounded background queue. + + Single use: once :meth:`aclose` has begun, :meth:`notify` refuses every event and :meth:`start` + raises ``RuntimeError`` (vault BACKLOG #3120). Build a new notifier rather than restart one.""" def __init__( self, @@ -579,12 +582,19 @@ def __init__( # The queue-full records still being written, held so the loop keeps them alive and so # aclose() can wait for them. self._recording: set[asyncio.Task[None]] = set() + # Set when aclose() begins, and never cleared: start() refuses (vault BACKLOG #3120). An event + # queued after that sits behind the stop sentinel, or on a queue nothing reads, and is never + # sent, so notify() refuses it; the contract on ``SecurityNotifier`` says why. + self._closing = False # Logged now, once, rather than at the first send (BACKLOG #1131): the first send may be a # lock notice, and a line appearing then would show a logs:view reader that a lock landed. if use_tls and not tls_verify: warn_smtp_verification_off(cell=ALERTS_SMTP_CELL, host=host) - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: + # Returns whether the event was queued for sending, the contract on ``SecurityNotifier``, + # which lists why it may answer False. Never a delivery receipt. + # # No deliverable address means nothing to email. The caller has audited the event; which # events the /me/security-events feed shows is stated once, in auth/notifications.py. # Non-blocking enqueue. @@ -629,23 +639,51 @@ async def notify(self, event: SecurityEvent) -> None: event.event_type, event.username, ) - return - # Read before the enqueue, with no await between, so it is the queue the enqueue meets. - full = self._queue.full() - self._enqueue( + return False + if self._closing: + # At shutdown, for example a lock notice the teardown's cut drain left running (vault + # BACKLOG #3120). Recorded like a full queue's drop. A record started after aclose() + # has finished is not waited for, and may meet the closed store, where it fails and is + # swallowed like any other. + if event.event_type not in LOG_SILENT_EVENT_TYPES: + log.warning( + "%s is closing; dropping %s for %s", + type(self).__name__, + notice_kind_log_label(event.event_type), + scrub_log_argument(event.username), + ) + self._record_dropped(event, "closing") + return False + queued = self._enqueue( event, dropped=None if event.event_type in LOG_SILENT_EVENT_TYPES else f"{event.event_type} for {event.username}", ) - if full and self._audit is not None and len(self._recording) < _MAX_RECORDING: - # Off the caller's path: a notice must never block a sign-in or an admin action, and - # this write would wait on the store under exactly the overload that filled the queue. - task = asyncio.create_task(self._record_undelivered(event, {"reason": "queue_full"})) - self._recording.add(task) - task.add_done_callback(self._recording.discard) + if not queued: + self._record_dropped(event, "queue_full") + return queued + + def _record_dropped(self, event: SecurityEvent, reason: str) -> None: + # Off the caller's path: a notice must never block a sign-in or an admin action, and this + # write would wait on the store under exactly the overload that filled the queue. + if self._audit is None or len(self._recording) >= _MAX_RECORDING: + return + task = asyncio.create_task(self._record_undelivered(event, {"reason": reason})) + self._recording.add(task) + task.add_done_callback(self._recording.discard) + + def start(self) -> None: + # Never again once closed (vault BACKLOG #3120). A close the teardown cut short can leave its + # stop sentinel on the queue, so a drain loop started again would read it and stop, and + # notify() would answer True for events nothing sends. The engine builds a new notifier for + # each lifespan rather than restart one. + if self._closing: + raise RuntimeError(f"{type(self).__name__} has been closed and cannot be started again") + super().start() async def aclose(self) -> None: + self._closing = True # before the sentinel goes in, so nothing is queued behind it try: await super().aclose() finally: diff --git a/messagefoundry/store/base.py b/messagefoundry/store/base.py index ba25d1093..b4ced4bef 100644 --- a/messagefoundry/store/base.py +++ b/messagefoundry/store/base.py @@ -2855,7 +2855,9 @@ async def list_sessions( async def touch_session(self, token_hash: str, *, now: float | None = None) -> None: ... - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session to ``new_token_hash``, in place (ASVS 7.2.4). A pure re-key: every other column — ``user_id``, ``created_at``, ``expires_at``, ``client``, @@ -2875,6 +2877,13 @@ async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: that report what they changed: every other one is deliberately blind (a write against a dead hash is a silent no-op), but a caller rotating a session is about to hand the new token to a user, so it must be able to fail closed if the session died underneath it. + + **A re-key that lands also records a lineage link, in the SAME transaction** (vault BACKLOG + #3028, ADR 0210): ``token_hash`` retired, replaced by ``new_token_hash``, stamped with + ``now`` or, when that is ``None``, the time the re-key landed, after any wait. The + session row itself carries nothing of its old hash, so the old hash still resolves to + nothing. The link's one reader is :meth:`supersede_session`; ``store/session_lineage.py`` + says why it is unsealed. A re-key that did not land writes no link. """ ... @@ -2904,21 +2913,36 @@ async def mark_session_mfa_verified( async def revoke_session(self, token_hash: str, *, now: float | None = None) -> None: ... - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke the unrevoked session ``token_hash`` names, and return its row as revoked. - - Returns ``None`` when no unrevoked row has that hash, so nothing was written. A row that is - expired or idle is still revoked and returned. Its other columns are as the revoke found - them, so the caller can judge whether the session was live, and ``revoked_at`` is ``now``. - - **ONE atomic operation, so it cannot interleave with :meth:`rotate_session` (BACKLOG - #2146).** It is one statement on Postgres and SQL Server, and one transaction on SQLite. The - login supersession used to read the row and then revoke it by hash, and a rotation that - re-keyed the row between the two left the revoke matching nothing, so the session lived on - under its new hash. Here the store orders the two writes on the row. If this one runs first, - the row is revoked and the rotation's ``revoked_at IS NULL`` guard refuses it, so the - rotation fails closed. If the rotation committed first, the hash names no row and this - returns ``None``. + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke the unrevoked session ``token_hash`` names, or the one it became, and return its + row as revoked. + + Returns ``None`` when nothing was revoked. A row that is expired or idle is still revoked + and returned. Its other columns are as the revoke found them, so the caller can judge + whether the session was live, and ``revoked_at`` is ``now``. Its ``token_hash`` is the hash + the session ended under, which differs from ``token_hash`` when a link was followed. + + **The revoke and the read-back are one write on the row, so they cannot interleave with + :meth:`rotate_session` (BACKLOG #2146).** The login supersession used to read the row and + then revoke it by hash, and a rotation that re-keyed the row between the two left the + revoke matching nothing. Here the store orders the two writes on the row. If this one runs + first, the row is revoked and the rotation's ``revoked_at IS NULL`` guard refuses it, so + the rotation fails closed. + + **A rotation that committed FIRST is followed, when ``lineage_since`` is given (vault + BACKLOG #3028, ADR 0210).** The presented hash then names no row, so the revoke steps along + the links :meth:`rotate_session` wrote, up to ``LINEAGE_MAX_HOPS`` of them, and ends the + live end, inside one transaction. A link retired before ``lineage_since`` is not followed, + so a retired hash leads anywhere only for a bounded time; the caller chooses the bound, + because the store reads no settings. On Postgres and SQL Server a rotation can still commit + between two steps; it costs the walk one more step, never the revoke, because each step + revokes before it reads a link. ``store/session_lineage.py`` says why that order is the + control. With + ``lineage_since`` left ``None`` no link is followed, and a rotation that committed first + leaves this returning ``None``, as before. The lineage lookup lives here and nowhere else: + no session READ follows a link, so a retired hash authenticates nothing. """ ... @@ -2984,12 +3008,20 @@ async def enforce_session_cap( ... async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: """Delete session rows past their absolute expiry, revoked or not, and return the count. With ``idle_seconds`` given, rows idle for longer are deleted too (BACKLOG #2096): the validator refuses them on presentation, so keeping them only grows the table. + With ``lineage_seconds`` given, session lineage links retired more than that long before + ``now`` are deleted too (vault BACKLOG #3028). The supersession no longer follows them, so + keeping them only grows the table. They are not counted in the return. + The idle variant scans the table, because its ``OR`` reaches past the expiry index, and that stands on purpose (BACKLOG #2283). The scan runs once per reaper pass over a table the purge itself keeps small. The alternative is an index on ``last_used_at``, and every diff --git a/messagefoundry/store/postgres.py b/messagefoundry/store/postgres.py index f3f507e55..390e43516 100644 --- a/messagefoundry/store/postgres.py +++ b/messagefoundry/store/postgres.py @@ -182,6 +182,7 @@ new_state_cache, sealed_reference_set, ) +from messagefoundry.store.session_lineage import revoke_through_lineage from messagefoundry.store.store import ( _ACTIVE_ALERT_STATUS_SQL, _ALERT_SEVERITY_RANK_SQL, @@ -744,6 +745,18 @@ def __init__(self, method: str, outbox_ids: tuple[str, ...]) -> None: "CREATE INDEX IF NOT EXISTS ix_audit_ts ON audit_log(ts)", # recent_audit_of: the start's config baseline read (vault BACKLOG #2597). "CREATE INDEX IF NOT EXISTS ix_audit_action ON audit_log(action, id)", + # The lock notice's throttle read, one account's rows of one action (vault BACKLOG #3120). Keyed + # on a PREFIX of the actor, because `actor` is unbounded TEXT and a btree entry over about 2.7 KB + # fails the INSERT: a sign-in can audit a typed username of any length, and an index on the whole + # value would turn an overlong one into a failed audit write. `_audit_where` filters on the same + # prefix and then on the whole value, so the planner can seek it. 200 characters is at most + # 800 bytes. + "CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(left(actor, 200), action, id)", + # One account's rows of any action, newest first: GET /audit?actor=, its export pages and + # /me/security-events (vault BACKLOG #3120). The SQLite `_SCHEMA` says why ix_audit_actor alone + # is not enough there. The same actor prefix, for the same reason; `_audit_where` and + # `_security_events_where_pg` both filter on it. + "CREATE INDEX IF NOT EXISTS ix_audit_actor_id ON audit_log(left(actor, 200), id)", # No table beside audit_log says where its keying starts: the process that holds the key decides, # and the chain's own row at sequence number 1 (the genesis row) names the first range's key. # Per-key AES-GCM invocation bound (ASVS 11.3.4) — see the SQLite `_SCHEMA` for the @@ -933,6 +946,15 @@ def __init__(self, method: str, outbox_ids: tuple[str, ...]) -> None: )""", "CREATE INDEX IF NOT EXISTS ix_sessions_user ON sessions(user_id)", "CREATE INDEX IF NOT EXISTS ix_sessions_expires ON sessions(expires_at)", + # Which token hash replaced a retired one (vault BACKLOG #3028, ADR 0210). The SQLite `_SCHEMA` + # says who reads it and why it is unsealed. Adding this DDL moves _schema_hash() automatically + # -- the ADR 0064 bump. + """CREATE TABLE IF NOT EXISTS session_lineage ( + retired_hash TEXT PRIMARY KEY, + successor_hash TEXT NOT NULL, + retired_at DOUBLE PRECISION NOT NULL + )""", + "CREATE INDEX IF NOT EXISTS ix_session_lineage_retired ON session_lineage(retired_at)", # WebAuthn passkeys (WP-14b, ADR 0068 §4). credential_id_hash — sha256 hex of the raw credential # id (the sessions.token_hash precedent) — is the parity-safe PK on all 3 backends; sign_count is # BIGINT (a WebAuthn uint32 overflows signed INT); label is column-capped so ux_webauthn_label @@ -1193,6 +1215,15 @@ def _pg_session_cap_keep_sql(split_mfa_pending: bool) -> str: " client=$6, reauth_at=$7, mfa_verified_at=$8, auth_mechanism=$9, idp_auth_time=$10," " user_epoch=$11, seal=$12 WHERE token_hash=$13 AND seal=$14" ) +# Vault BACKLOG #3028: store.py's three session lineage statements respelled for `$n`. store.py says +# which method runs each, and that no other statement names the table. +_PG_SESSION_LINEAGE_INSERT: Final = ( + "INSERT INTO session_lineage (retired_hash, successor_hash, retired_at) VALUES ($1,$2,$3)" +) +_PG_SESSION_LINEAGE_SUCCESSOR: Final = ( + "SELECT successor_hash FROM session_lineage WHERE retired_hash=$1 AND retired_at >= $2" +) +_PG_SESSION_LINEAGE_PURGE: Final = "DELETE FROM session_lineage WHERE retired_at < $1" #: The whole-account session sweep: revoke and blank the seal (BACKLOG #2592). Each caller raises #: the account's session epoch in the same transaction. _PG_REVOKE_USER_SESSIONS_SQL: Final = ( @@ -8053,7 +8084,10 @@ def _audit_where( params: list[Any] = [] if actor is not None: params.append(actor) - clauses.append(f"actor = ${len(params)}") + # The prefix term lets the planner seek ix_audit_actor and ix_audit_actor_id, which are + # keyed on that prefix (vault BACKLOG #3120); the whole-value term decides the match. + n = len(params) + clauses.append(f"left(actor, 200) = left(${n}::text, 200) AND actor = ${n}") if action is not None: params.append(action) clauses.append(f"action = ${len(params)}") @@ -9852,16 +9886,25 @@ async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None async with self._timed_acquire(record=False) as conn, conn.transaction(): await self._write_session(conn, token_hash, stamp) - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session in place (ASVS 7.2.4). See :meth:`AuthStore.rotate_session`. The token hash is sealed, so the re-key goes through :meth:`_write_session` (BACKLOG - #2592). The answer is read after the transaction has committed. The borrow is + #2592). The lineage link goes in the same transaction, only when the re-key landed (vault + BACKLOG #3028), its time read once the re-key has landed, after any wait for the borrow + or the row lock. The answer is read after the transaction has committed. The borrow is :meth:`_after_commit_txn`'s, for the reason it gives.""" async with self._after_commit_txn() as conn: rotated = await self._write_session( conn, token_hash, lambda s: replace(s, token_hash=new_token_hash) ) + if rotated is not None: + retired_at = time.time() if now is None else now + await conn.execute( + _PG_SESSION_LINEAGE_INSERT, token_hash, new_token_hash, retired_at + ) return rotated is not None async def revoke_session(self, token_hash: str, *, now: float | None = None) -> None: @@ -9872,15 +9915,33 @@ async def revoke_session(self, token_hash: str, *, now: float | None = None) -> token_hash, ) - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke and return in one statement (BACKLOG #2146). See :meth:`AuthStore.supersede_session`.""" - d = await self._fetchone( - "UPDATE sessions SET revoked_at=$1, seal='' WHERE token_hash=$2 AND revoked_at IS NULL" - " RETURNING *", - now, - token_hash, - ) - return SessionRecord.from_mapping(dict(d)) if d else None + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke and return, one statement per step, in one transaction (BACKLOG #2146, vault + BACKLOG #3028). See :meth:`AuthStore.supersede_session`. + + Each step's ``UPDATE ... RETURNING`` locks the row it ends. Under READ COMMITTED each + statement reads what has committed when it starts, so a rotation that commits while the + walk runs is seen by the next step's lineage read. The borrow stays bounded, as it was.""" + async with self._timed_acquire(record=False) as conn, conn.transaction(): + + async def _revoke(h: str) -> SessionRecord | None: + d = await conn.fetchrow( + "UPDATE sessions SET revoked_at=$1, seal='' WHERE token_hash=$2" + " AND revoked_at IS NULL RETURNING *", + now, + h, + ) + return SessionRecord.from_mapping(dict(d)) if d else None + + async def _successor(h: str, since: float) -> str | None: + found = await conn.fetchval(_PG_SESSION_LINEAGE_SUCCESSOR, h, since) + return None if found is None else str(found) + + return await revoke_through_lineage( + token_hash, lineage_since=lineage_since, revoke=_revoke, successor=_successor + ) async def revoke_user_sessions( self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None @@ -9941,19 +10002,27 @@ async def enforce_session_cap( ) async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: now = time.time() if now is None else now # Through `_execute`, the bounded borrow, rather than `self._pool.execute`, which acquires # with no timeout (BACKLOG #1052). `_execute` borrows with `record=False`, which keeps this # hourly sweep out of the worker acquire-wait curve, and returns the count (BACKLOG #2283). if idle_seconds is None: - return await self._execute("DELETE FROM sessions WHERE expires_at < $1", now) - return await self._execute( - "DELETE FROM sessions WHERE expires_at < $1 OR $1 - last_used_at > $2", - now, - float(idle_seconds), - ) + purged = await self._execute("DELETE FROM sessions WHERE expires_at < $1", now) + else: + purged = await self._execute( + "DELETE FROM sessions WHERE expires_at < $1 OR $1 - last_used_at > $2", + now, + float(idle_seconds), + ) + if lineage_seconds is not None: + await self._execute(_PG_SESSION_LINEAGE_PURGE, now - float(lineage_seconds)) + return purged # --- retention / purge + maintenance (PHI.md §8) ------------------------- @@ -10548,10 +10617,13 @@ def _pg_cutoff_case( def _security_events_where_pg(username: str, until: float | None) -> tuple[str, list[Any]]: """The ``$N`` ``WHERE`` text and values for one user's security-event page and its total (BACKLOG #2438), shared so the two count one set. The SQLite builder states the rule, including - the lower bound at the account's ``created_at`` (BACKLOG #1152).""" + the lower bound at the account's ``created_at`` (BACKLOG #1152). + + The actor prefix term lets the planner seek ``ix_audit_actor_id``, which is keyed on that + prefix, as ``_audit_where`` does (vault BACKLOG #3120); the whole-value term decides the match.""" params: list[Any] = [username] clause = ( - " WHERE actor = $1 AND action LIKE 'auth.%'" + " WHERE left(actor, 200) = left($1::text, 200) AND actor = $1 AND action LIKE 'auth.%'" " AND ts >= (SELECT created_at FROM users WHERE username = $1)" ) if until is not None: diff --git a/messagefoundry/store/session_lineage.py b/messagefoundry/store/session_lineage.py new file mode 100644 index 000000000..8b3e3f84a --- /dev/null +++ b/messagefoundry/store/session_lineage.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""The session lineage: which token hash replaced a retired one (vault BACKLOG #3028, ADR 0210). + +A rotation re-keys a session row to a new token hash, so the old hash stops naming any row. Each +backend's ``rotate_session`` now writes one ``session_lineage`` row in the same transaction: the +retired hash, the hash that replaced it, and when. This module holds the one walk over those rows +that every backend shares, so the three cannot disagree on how far it goes. + +**A retired hash authenticates nothing.** No session read consults this table: ``get_session``, +``list_sessions`` and the service's session loader read ``sessions`` only. The one reader is the +revoke below, so the only thing a retired hash can lead to is the END of the session it became. + +The rows are NOT sealed under BACKLOG #2592, deliberately. A seal stops a database writer forging +authority, and a link grants none: the most a forged link can do is direct a revoke at a live row, +and a revoke needs no key on any backend (it blanks the row's seal). A writer who can forge a link +can already revoke that row directly. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: # store.py imports this module, so the record type is for the checker only + from messagefoundry.store.store import SessionRecord + +#: How many links one revoke follows before it gives up and ends nothing. A link is written only by +#: a completed elevation, and every link it follows is inside the caller's window, so a chain this +#: long means that many credential ceremonies on one session inside that window. Bounded so a +#: revoke's work is bounded whatever the table holds. +LINEAGE_MAX_HOPS: Final = 32 + +#: Revoke the unrevoked row that hash names and return it as revoked, or ``None`` when none did. +type Revoke = Callable[[str], Awaitable[SessionRecord | None]] +#: The hash that replaced this one, by a link retired at or after ``since``, or ``None``. +type Successor = Callable[[str, float], Awaitable[str | None]] + + +async def revoke_through_lineage( + token_hash: str, + *, + lineage_since: float | None, + revoke: Revoke, + successor: Successor, +) -> SessionRecord | None: + """Revoke the session ``token_hash`` names or, when it names none, the session it BECAME. + + Run it inside ONE store transaction, with ``revoke`` and ``successor`` bound to that + transaction. Each step first tries to revoke the hash it holds; only when that revokes nothing + does it read the hash's successor and step to it. So a rotation that commits between a + ``successor`` read and the next ``revoke`` costs one more step rather than the revoke: the + re-keyed row's old hash then has a link of its own, written in the rotation's transaction. + **That order is the control on Postgres and SQL Server, not the transaction:** under READ + COMMITTED a rotation can commit between two steps. Reading the whole chain first and revoking + its end afterwards would aim the revoke at a hash such a rotation had already retired. + + ``lineage_since`` is the oldest ``retired_at`` a followed link may carry. ``None`` follows no + link at all, which is the plain revoke of the hash given (BACKLOG #2146's behaviour). + + Returns the row it revoked, whose ``token_hash`` is the hash it ended under, or ``None`` when + the walk reached a hash with no unrevoked row and no live link, or ran out of hops.""" + current = token_hash + for hop in range(LINEAGE_MAX_HOPS + 1): + ended = await revoke(current) + if ended is not None or lineage_since is None or hop == LINEAGE_MAX_HOPS: + return ended + following = await successor(current, lineage_since) + if following is None: + return None + current = following + return None diff --git a/messagefoundry/store/sqlserver.py b/messagefoundry/store/sqlserver.py index ce220d4e2..483ddb26f 100644 --- a/messagefoundry/store/sqlserver.py +++ b/messagefoundry/store/sqlserver.py @@ -166,12 +166,16 @@ new_state_cache, sealed_reference_set, ) +from messagefoundry.store.session_lineage import revoke_through_lineage from messagefoundry.store.store import ( _ACTIVE_ALERT_STATUS_SQL, _ALERT_SEVERITY_RANK_SQL, _SESSION_CAP_ORDER_SQL, _SESSION_CAP_RANK_NOT_AHEAD_SQL, _SESSION_INSERT, + _SESSION_LINEAGE_INSERT, + _SESSION_LINEAGE_PURGE, + _SESSION_LINEAGE_SUCCESSOR, _SESSION_LIVE_SQL, _SESSION_NOT_AHEAD_SQL, _SQL_ADMIN_GUARD_HOLDERS, @@ -2150,6 +2154,15 @@ def __init__(self, conn: Any, cur: Any) -> None: # recent_audit_of: the start's config baseline read (vault BACKLOG #2597). """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_action','IndexID') IS NULL CREATE INDEX ix_audit_action ON audit_log(action, id)""", + # The lock notice's throttle read, one account's rows of one action (vault BACKLOG #3120). The + # key is at most 772 bytes (actor 512, action 256, id 4), inside the 1700-byte nonclustered limit. + """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_actor','IndexID') IS NULL + CREATE INDEX ix_audit_actor ON audit_log(actor, action, id)""", + # One account's rows of any action, newest first: GET /audit?actor=, its export pages and + # /me/security-events (vault BACKLOG #3120). The SQLite `_SCHEMA` says why ix_audit_actor alone + # is not enough there. The key is at most 516 bytes (actor 512, id 4). + """IF INDEXPROPERTY(OBJECT_ID('audit_log'),'ix_audit_actor_id','IndexID') IS NULL + CREATE INDEX ix_audit_actor_id ON audit_log(actor, id)""", # No table beside audit_log says where its keying starts: the process that holds the key decides, # and the chain's own row at sequence number 1 (the genesis row) names the first range's key. # Per-key AES-GCM invocation bound (ASVS 11.3.4) — see the SQLite `_SCHEMA` for the @@ -2453,6 +2466,14 @@ def __init__(self, conn: Any, cur: Any) -> None: CREATE INDEX ix_sessions_user ON sessions(user_id)""", """IF INDEXPROPERTY(OBJECT_ID('sessions'),'ix_sessions_expires','IndexID') IS NULL CREATE INDEX ix_sessions_expires ON sessions(expires_at)""", + # Which token hash replaced a retired one (vault BACKLOG #3028, ADR 0210). The SQLite `_SCHEMA` + # says who reads it and why it is unsealed. Both hashes are 64-character hex digests, the + # sessions.token_hash width. Adding this DDL moves _schema_hash() -- the ADR 0064 bump. + """IF OBJECT_ID('session_lineage','U') IS NULL CREATE TABLE session_lineage ( + retired_hash NVARCHAR(64) NOT NULL PRIMARY KEY, + successor_hash NVARCHAR(64) NOT NULL, retired_at FLOAT NOT NULL)""", + """IF INDEXPROPERTY(OBJECT_ID('session_lineage'),'ix_session_lineage_retired','IndexID') IS NULL + CREATE INDEX ix_session_lineage_retired ON session_lineage(retired_at)""", # WebAuthn passkeys (WP-14b, ADR 0068 §4). credential_id_hash — sha256 hex of the RAW credential # id (the sessions.token_hash precedent) — is the PK: raw WebAuthn ids may be up to 1023 bytes, # unboundable as an index key here (NVARCHAR(MAX) can't be a PK/index key), so the fixed-width @@ -14293,15 +14314,19 @@ async def _write_session_txn( token_hash: str, change: Callable[[SessionRecord], SessionRecord], *, + landed: Callable[[Any], Awaitable[None]] | None = None, after_commit: bool = False, ) -> SessionRecord | None: """:meth:`_write_session` in a transaction of its own, committed before it returns. + ``landed`` runs on the same cursor, in the same transaction, only when the write landed. ``after_commit`` borrows through :meth:`_acquire_after_commit`; only ``rotate_session`` passes it.""" borrow = self._acquire_after_commit() if after_commit else self._acquire() async with borrow as conn, self._cursor(conn) as cur: try: updated = await self._write_session(cur, token_hash, change) + if updated is not None and landed is not None: + await landed(cur) await self._commit(conn) except Exception: await conn.rollback() @@ -14357,16 +14382,28 @@ async def mark_session_reauthed( async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None = None) -> None: await self._write_session_txn(token_hash, mfa_stamp(time.time() if now is None else now)) - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session in place (ASVS 7.2.4). See :meth:`AuthStore.rotate_session`. The token hash is sealed, so the re-key goes through :meth:`_write_session` (BACKLOG #2592), whose write reads an ``OUTPUT`` rowset and never ``cursor.rowcount`` (BACKLOG #2283). ``token_hash`` is ``NVARCHAR(64)`` and a hex digest is exactly 64 chars, so the - clustered-PK row move is width-safe. The borrow is :meth:`_acquire_after_commit`'s, for the - reason it gives (BACKLOG #3023).""" + clustered-PK row move is width-safe. The lineage link goes in the same transaction, only + when the re-key landed (vault BACKLOG #3028), its time read then, after any wait for the + borrow or the row lock. The borrow is :meth:`_acquire_after_commit`'s, for the reason it + gives (BACKLOG #3023).""" + + async def _link(cur: Any) -> None: + retired_at = time.time() if now is None else now + await cur.execute(_SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, retired_at)) + rotated = await self._write_session_txn( - token_hash, lambda s: replace(s, token_hash=new_token_hash), after_commit=True + token_hash, + lambda s: replace(s, token_hash=new_token_hash), + landed=_link, + after_commit=True, ) return rotated is not None @@ -14377,17 +14414,41 @@ async def revoke_session(self, token_hash: str, *, now: float | None = None) -> (now, token_hash), ) - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke and return in one statement (BACKLOG #2146). See :meth:`AuthStore.supersede_session`. + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke and return, one statement per step, in one transaction (BACKLOG #2146, vault + BACKLOG #3028). See :meth:`AuthStore.supersede_session`. - Through ``_execute_output`` rather than ``_fetchone``, which commits as a READ, while this is - a durable write (BACKLOG #2283).""" - rows = await self._execute_output( - "UPDATE sessions SET revoked_at=?, seal='' OUTPUT inserted.*" - " WHERE token_hash=? AND revoked_at IS NULL", - (now, token_hash), - ) - return SessionRecord.from_mapping(rows[0]) if rows else None + Each step's ``UPDATE ... OUTPUT`` takes the row's lock, and its rowset says what it ended, + which a session-wide ``SET NOCOUNT ON`` cannot hide (BACKLOG #2283). Each statement reads + what has committed when it runs, so a rotation that commits while the walk runs is seen by + the next step's lineage read. Committed as a durable write, never as a READ.""" + async with self._acquire() as conn, self._cursor(conn) as cur: + + async def _revoke(h: str) -> SessionRecord | None: + await cur.execute( + "UPDATE sessions SET revoked_at=?, seal='' OUTPUT inserted.*" + " WHERE token_hash=? AND revoked_at IS NULL", + (now, h), + ) + rows = await self._fetch_dicts(cur) + return SessionRecord.from_mapping(rows[0]) if rows else None + + async def _successor(h: str, since: float) -> str | None: + await cur.execute(_SESSION_LINEAGE_SUCCESSOR, (h, since)) + rows = await cur.fetchall() + return str(rows[0][0]) if rows else None + + try: + ended = await revoke_through_lineage( + token_hash, lineage_since=lineage_since, revoke=_revoke, successor=_successor + ) + await self._commit(conn) + except Exception: + await conn.rollback() + raise + return ended async def revoke_user_sessions( self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None @@ -14462,7 +14523,11 @@ async def enforce_session_cap( return len(rows) async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: now = time.time() if now is None else now if idle_seconds is None: @@ -14476,7 +14541,12 @@ async def purge_expired_sessions( # `revoke_user_sessions` this keeps the driver's count, which a session-wide SET NOCOUNT ON # can turn to -1: the reaper ignores it and decides nothing on it, and an OUTPUT rowset # would carry every purged row back each pass for a number nobody acts on. - return await self._execute(sql, params) + purged = await self._execute(sql, params) + if lineage_seconds is not None: + # Its own statement and transaction, as on Postgres: a link is table hygiene, and the + # supersession stops following one at its window whether or not this has run. + await self._execute(_SESSION_LINEAGE_PURGE, (now - float(lineage_seconds),)) + return purged async def stats(self) -> dict[str, int]: rows = await self._fetchall( diff --git a/messagefoundry/store/store.py b/messagefoundry/store/store.py index e26968ea4..a9e46904f 100644 --- a/messagefoundry/store/store.py +++ b/messagefoundry/store/store.py @@ -184,6 +184,7 @@ new_state_cache, sealed_reference_set, ) +from messagefoundry.store.session_lineage import revoke_through_lineage from messagefoundry.store.transit_attestation import ( TRANSIT_BOUND_ATTESTED_ACTION, TRANSIT_BOUND_WITHDRAWN_ACTION, @@ -1279,6 +1280,18 @@ class ConnectionEventWrite(TypedDict): " seal=? WHERE token_hash=? AND seal=?" ) +# Vault BACKLOG #3028, ADR 0210: the session lineage's three statements, in `?` form for SQLite and +# SQL Server; postgres.py respells them for `$n`. The INSERT runs only inside rotate_session's own +# transaction, the SELECT only inside supersede_session's, and the DELETE only from the session +# purge. No other statement names the table: a retired hash must authenticate nothing. +_SESSION_LINEAGE_INSERT: Final = ( + "INSERT INTO session_lineage (retired_hash, successor_hash, retired_at) VALUES (?,?,?)" +) +_SESSION_LINEAGE_SUCCESSOR: Final = ( + "SELECT successor_hash FROM session_lineage WHERE retired_hash=? AND retired_at >= ?" +) +_SESSION_LINEAGE_PURGE: Final = "DELETE FROM session_lineage WHERE retired_at < ?" + def session_reseal_params(new: SessionRecord, old: SessionRecord) -> tuple[object, ...]: """The parameters :data:`_SESSION_RESEAL` binds, in order: ``new``'s sealed columns and seal, @@ -6982,6 +6995,13 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] -- The newest row of a few actions (recent_audit_of): the start's config baseline read, vault BACKLOG -- #2597. Without it that read walks the whole chain newest-first until one of the actions turns up. CREATE INDEX IF NOT EXISTS ix_audit_action ON audit_log(action, id); +-- One account's rows of one action, newest first: the lock notice's 24-hour throttle read, vault +-- BACKLOG #3120. Without it that read walks every recent row of the action, for every account. +CREATE INDEX IF NOT EXISTS ix_audit_actor ON audit_log(actor, action, id); +-- One account's rows of any action, newest first: GET /audit?actor=, its export pages and +-- /me/security-events, vault BACKLOG #3120. ix_audit_actor alone made the planner seek the actor +-- there and then sort every one of its rows to read the newest few. +CREATE INDEX IF NOT EXISTS ix_audit_actor_id ON audit_log(actor, id); -- There is no table beside audit_log that says where its keying starts. Whether a chain is keyed is -- decided by the process that holds the key, and the first range's key is named by the chain's own @@ -7190,6 +7210,17 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] CREATE INDEX IF NOT EXISTS ix_sessions_user ON sessions(user_id); CREATE INDEX IF NOT EXISTS ix_sessions_expires ON sessions(expires_at); +-- Which token hash replaced a retired one, written by rotate_session in its own transaction (vault +-- BACKLOG #3028, ADR 0210). Read by the login supersession ONLY, never by a session read, so a +-- retired hash authenticates nothing. Not sealed: store/session_lineage.py says why. No FK, and the +-- session reaper purges rows past the window the supersession follows them for. +CREATE TABLE IF NOT EXISTS session_lineage ( + retired_hash TEXT PRIMARY KEY, -- the hash the rotation retired + successor_hash TEXT NOT NULL, -- the hash the session carried next + retired_at REAL NOT NULL -- when the re-key landed, in its transaction (wall clock) +); +CREATE INDEX IF NOT EXISTS ix_session_lineage_retired ON session_lineage(retired_at); + CREATE TABLE IF NOT EXISTS webauthn_credentials ( credential_id_hash TEXT PRIMARY KEY, -- sha256 hex of the RAW credential id (sessions.token_hash -- precedent): WebAuthn ids may be up to 1023 raw bytes — @@ -7218,7 +7249,7 @@ def _dead_target_pairs(rows: Iterable[tuple[Any, Any]]) -> list[tuple[str, str]] -- and client host where a sign-in finished owing nothing more, or a step-up or enrolment passed -- (AuthService._mark_login_address_known says when). Keyed on the account id, never the -- username, so a re-created namesake inherits nothing. The primary key serves the signal's one read --- (WHERE user_id), so audit_log, which has no actor index, is never scanned for it. Plaintext like +-- (WHERE user_id), so audit_log is never read for it. Plaintext like -- sessions.client: a network address, not PHI (docs/PHI.md section 2). ON DELETE CASCADE because -- the write is best-effort and may race delete_user: a row inserted just before the account's -- delete goes with it, and one inserted after fails the key and is dropped by the caller. @@ -16147,17 +16178,26 @@ async def mark_session_mfa_verified(self, token_hash: str, *, now: float | None await self._write_session(token_hash, stamp) await self._commit() - async def rotate_session(self, token_hash: str, *, new_token_hash: str) -> bool: + async def rotate_session( + self, token_hash: str, *, new_token_hash: str, now: float | None = None + ) -> bool: """Re-key a live session in place (ASVS 7.2.4). See :meth:`AuthStore.rotate_session`. ``token_hash`` is the PK, so this is a key update: the row keeps every other column and the old hash stops resolving in the same transaction. The token hash is sealed, so the re-key re-seals (BACKLOG #2592). Reports whether a row was re-keyed so the caller can fail closed - when the session was revoked out from under it.""" + when the session was revoked out from under it. The lineage link is written in the same + transaction, and only when the re-key landed (vault BACKLOG #3028). Its time is read once + the re-key has landed, so a wait for the lock cannot age the link before it exists.""" async with _writer_txn(self._db, self._lock): rotated = await self._write_session( token_hash, lambda s: replace(s, token_hash=new_token_hash) ) + if rotated is not None: + retired_at = time.time() if now is None else now + await self._db.execute( + _SESSION_LINEAGE_INSERT, (token_hash, new_token_hash, retired_at) + ) await self._commit() return rotated is not None @@ -16170,28 +16210,41 @@ async def revoke_session(self, token_hash: str, *, now: float | None = None) -> ) await self._commit() - async def supersede_session(self, token_hash: str, *, now: float) -> SessionRecord | None: - """Revoke, then read the row back in the SAME transaction (BACKLOG #2146). See + async def supersede_session( + self, token_hash: str, *, now: float, lineage_since: float | None = None + ) -> SessionRecord | None: + """Revoke, then read the row back in the SAME transaction (BACKLOG #2146), following the + session lineage when ``lineage_since`` is given (vault BACKLOG #3028). See :meth:`AuthStore.supersede_session`. - Two statements here rather than one ``UPDATE ... RETURNING``, because ``RETURNING`` needs - SQLite 3.35 and a Linux Python can link an older system library. They are still one unit: - the writer lock orders them against every in-process writer, ``rotate_session`` included, - and the write lock the ``UPDATE`` takes holds until the commit, so no other process can - re-key the row between them.""" - async with _writer_guard(self._db, self._lock): + Two statements per step rather than one ``UPDATE ... RETURNING``, because ``RETURNING`` + needs SQLite 3.35 and a Linux Python can link an older system library. They are still one + unit: the writer lock orders them against every in-process writer, ``rotate_session`` + included. The first ``UPDATE`` takes SQLite's write lock even when it matches no row, and + holds it until the commit, so no other process can re-key a row while the walk runs.""" + + async def _revoke(h: str) -> SessionRecord | None: cur = await self._db.execute( "UPDATE sessions SET revoked_at=?, seal='' WHERE token_hash=? AND revoked_at IS NULL", - (now, token_hash), + (now, h), + ) + if not cur.rowcount: + return None + cur = await self._db.execute("SELECT * FROM sessions WHERE token_hash=?", (h,)) + row = await cur.fetchone() + return SessionRecord.from_mapping(dict(row)) if row else None + + async def _successor(h: str, since: float) -> str | None: + cur = await self._db.execute(_SESSION_LINEAGE_SUCCESSOR, (h, since)) + row = await cur.fetchone() + return str(row[0]) if row else None + + async with _writer_guard(self._db, self._lock): + ended = await revoke_through_lineage( + token_hash, lineage_since=lineage_since, revoke=_revoke, successor=_successor ) - row = None - if cur.rowcount: - cur = await self._db.execute( - "SELECT * FROM sessions WHERE token_hash=?", (token_hash,) - ) - row = await cur.fetchone() await self._commit() - return SessionRecord.from_mapping(dict(row)) if row else None + return ended async def revoke_user_sessions( self, user_id: str, *, except_token_hash: str | None = None, now: float | None = None @@ -16271,7 +16324,11 @@ async def enforce_session_cap( return int(cur.rowcount) async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> int: now = time.time() if now is None else now async with _writer_guard(self._db, self._lock): @@ -16282,8 +16339,11 @@ async def purge_expired_sessions( "DELETE FROM sessions WHERE expires_at < ? OR ? - last_used_at > ?", (now, now, float(idle_seconds)), ) + purged = cur.rowcount if cur.rowcount is not None else 0 + if lineage_seconds is not None: + await self._db.execute(_SESSION_LINEAGE_PURGE, (now - float(lineage_seconds),)) await self._commit() - return cur.rowcount if cur.rowcount is not None else 0 + return purged async def db_status(self) -> DbStatus: """Database health snapshot (size, free space, journal mode, row counts). diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py b/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py index b7c3ae662..efc5051b8 100644 --- a/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py +++ b/packaging/messagefoundry-webconsole/tests/test_ui_notify_address.py @@ -32,8 +32,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _service( diff --git a/tests/_session_rotation_contract.py b/tests/_session_rotation_contract.py index c0124cbca..57850a2a3 100644 --- a/tests/_session_rotation_contract.py +++ b/tests/_session_rotation_contract.py @@ -226,7 +226,7 @@ async def assert_session_supersession_contract(store: Any, *, user_id: str = "su # --- an unknown hash ------------------------------------------------------------------------ assert await store.supersede_session(_h(), now=now) is None - # --- the other order: a rotation that committed first leaves the old hash naming nothing ---- + # --- the other order, with no lineage window: a rotation that committed first wins --------- old, new = _h(), _h() await store.create_session(token_hash=old, user_id=user_id, expires_at=expires, now=now) assert await store.rotate_session(old, new_token_hash=new) is True @@ -235,3 +235,77 @@ async def assert_session_supersession_contract(store: Any, *, user_id: str = "su assert survivor is not None and survivor.revoked_at is None, ( "superseding a retired hash must not touch the session that now carries another hash" ) + + +async def assert_session_lineage_contract(store: Any, *, user_id: str = "lin-u1") -> None: + """Drive the session lineage through its contract on any backend (vault BACKLOG #3028). + + ``rotate_session`` links the hash it retired to its successor in its own transaction, and + ``supersede_session`` follows those links, inside a window its caller names, to the live end. + A retired hash still resolves to nothing, and the reaper's purge drops links past the window. + """ + now = time.time() + await store.create_user( + user_id=user_id, + username=f"lin-{user_id}", + auth_provider="local", + display_name=None, + email=None, + password_hash="h", + now=now, + password_generated=False, + ) + expires = now + 9876.5 + + # --- two rotations: a -> b -> c, and neither retired hash resolves ------------------------- + a, b, c = _h(), _h(), _h() + await store.create_session(token_hash=a, user_id=user_id, expires_at=expires, now=now) + assert await store.rotate_session(a, new_token_hash=b, now=now) is True + assert await store.rotate_session(b, new_token_hash=c, now=now + 1) is True + assert await store.get_session(a) is None, "a retired hash must resolve to nothing" + assert await store.get_session(b) is None, "a retired hash must resolve to nothing" + + # --- no window: the plain revoke of the hash given, as before ------------------------------ + assert await store.supersede_session(a, now=now + 2) is None + + # --- a window that starts after the first link: that link is not followed ------------------- + assert await store.supersede_session(a, now=now + 2, lineage_since=now + 0.5) is None + live = await store.get_session(c) + assert live is not None and live.revoked_at is None, "a link past the window was followed" + + # --- inside the window: followed over both hops, and the live end is what ends -------------- + ended = await store.supersede_session(a, now=now + 3, lineage_since=now - 10) + assert ended is not None, "the supersession did not follow the lineage to the live end" + assert ended.token_hash == c and ended.user_id == user_id and ended.revoked_at == now + 3 + stored = await store.get_session(c) + assert stored is not None and stored.revoked_at == now + 3, "the revoke was not persisted" + # The chain now ends at a revoked row, so a second supersession ends nothing. + assert await store.supersede_session(a, now=now + 4, lineage_since=now - 10) is None + # From the middle of the chain, too. + assert await store.supersede_session(b, now=now + 4, lineage_since=now - 10) is None + + # --- a rotation that did not land writes no link ------------------------------------------- + d = _h() + assert await store.rotate_session(c, new_token_hash=d, now=now + 5) is False + await store.create_session(token_hash=d, user_id=user_id, expires_at=expires, now=now) + assert await store.supersede_session(c, now=now + 6, lineage_since=now - 10) is None, ( + "a refused rotation linked its hash to a session it never became" + ) + kept = await store.get_session(d) + assert kept is not None and kept.revoked_at is None + + # --- the purge drops links past the window and keeps the rest ------------------------------ + old, old_next, fresh, fresh_next = _h(), _h(), _h(), _h() + await store.create_session(token_hash=old, user_id=user_id, expires_at=expires, now=now) + await store.create_session(token_hash=fresh, user_id=user_id, expires_at=expires, now=now) + assert await store.rotate_session(old, new_token_hash=old_next, now=now - 1000) is True + assert await store.rotate_session(fresh, new_token_hash=fresh_next, now=now) is True + await store.purge_expired_sessions(now=now) # no window: every link stays + await store.purge_expired_sessions(now=now, lineage_seconds=100) + assert await store.supersede_session(old, now=now, lineage_since=now - 10_000) is None, ( + "a link past the purge window survived the purge" + ) + survivor = await store.get_session(old_next) + assert survivor is not None and survivor.revoked_at is None + swept = await store.supersede_session(fresh, now=now + 7, lineage_since=now - 10) + assert swept is not None and swept.token_hash == fresh_next, "the purge dropped a live link" diff --git a/tests/test_ad_directory_identity.py b/tests/test_ad_directory_identity.py index 8721fd798..4cb1a0469 100644 --- a/tests/test_ad_directory_identity.py +++ b/tests/test_ad_directory_identity.py @@ -593,8 +593,9 @@ class _CapturingNotifier: def __init__(self) -> None: self.sent: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.sent.append(event) + return True async def test_a_login_rename_sends_the_holder_one_username_notice() -> None: diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 66a113e1e..dd7018e26 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -43,6 +43,7 @@ DirectoryProbe, LdapError, LdapReferralError, + LdapUnansweredError, ) from messagefoundry.auth.notifications import ( ACCOUNT_DISABLED, @@ -718,6 +719,11 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N ], reconcile.PROBE_FAULT_ABORT, ), + # BACKLOG #3010 shares it too: a pass of configuration refusals only. + ( + [reconcile.Probe("m", "m", reconcile.ProbeOutcome.MISCONFIGURED)], + reconcile.MISCONFIGURED_ABORT, + ), ], ids=[ "outage", @@ -725,6 +731,7 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N "referrals-and-failures", "faults-only", "faults-and-failures", + "refusals-only", ], ) def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( @@ -749,6 +756,61 @@ def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( assert [r.reason for r in ordinary.revocations] == ["directory_object_id_missing"] +def _probes(*pairs: tuple[str, reconcile.ProbeOutcome]) -> list[reconcile.Probe]: + return [reconcile.Probe(uid, uid, outcome) for uid, outcome in pairs] + + +_M, _R, _F, _U, _A = ( + reconcile.ProbeOutcome.MISCONFIGURED, + reconcile.ProbeOutcome.REFERRED, + reconcile.ProbeOutcome.FAULT, + reconcile.ProbeOutcome.UNAVAILABLE, + reconcile.ProbeOutcome.ABSENT, +) + + +@pytest.mark.parametrize( + ("pairs", "aborted"), + [ + ((("m", _M),), reconcile.MISCONFIGURED_ABORT), + ((("m", _M), ("u", _U)), reconcile.MISCONFIGURED_ABORT), + # A referral names the fix most closely, then the refusal; a fault names none. + ((("m", _M), ("f", _F)), reconcile.MISCONFIGURED_ABORT), + ((("m", _M), ("r", _R)), reconcile.REFERRAL_ABORT), + # THE CONTROL: failures alone are an outage, which pages nobody. + ((("u", _U), ("v", _U)), "directory_unavailable"), + ], + ids=["refusals", "refusals-and-failures", "beside-a-fault", "beside-a-referral", "outage"], +) +def test_a_pass_of_configuration_refusals_aborts_on_its_own_slug( + pairs: tuple[tuple[str, reconcile.ProbeOutcome], ...], aborted: str +) -> None: + """BACKLOG #3010. A deleted or moved search base refuses every probe with noSuchObject. Read as + an outage, that pass paged nobody; before #3010 it read as "not found" and struck everyone. It + aborts as ``directory_misconfigured``, which judges nothing and alerts, and never revokes. + + RED before D3b: there is no such outcome, and the refusal read as an outage.""" + plan = _plan(_probes(*pairs), prior_strikes={"m": 1, "u": 1}) + assert plan.aborted == aborted + assert plan.judged_nothing and plan.revocations == () and plan.strikes == {} + assert plan.directory_misconfigured is (aborted == reconcile.MISCONFIGURED_ABORT) + assert plan.misconfigured == tuple(uid for uid, o in pairs if o is _M) + + +def test_a_configuration_refusal_beside_answers_leaves_only_that_account_unjudged() -> None: + """Like a referral or a fault: the refused account keeps its strike and its recorded outcome, + and the rest of the pass is judged, revocations included.""" + plan = _plan( + _probes(("m", _M), ("gone", _A)), + prior_strikes={"m": 1, "gone": 1}, + prior_outcomes={"m": reconcile.ProbeOutcome.PRESENT}, + ) + assert plan.aborted is None and not plan.judged_nothing + assert plan.misconfigured == ("m",) + assert "m" not in plan.strikes and "m" not in plan.outcomes + assert [r.user_id for r in plan.revocations] == ["gone"] + + def test_breaker_ceiling_reports_the_larger_of_the_two_thresholds() -> None: assert reconcile.breaker_ceiling(probed=10, max_absolute=5, max_fraction=0.34) == 5 assert reconcile.breaker_ceiling(probed=300, max_absolute=5, max_fraction=0.34) == 102 @@ -1469,7 +1531,7 @@ async def test_when_every_probe_raises_the_operator_sees_a_probe_fault_not_an_ou "reason": "probe_fault", "faulted": 2, "unavailable": 0, - "probed": 2, + "asked": 2, "aborted": "probe_fault", } assert _faulted_rows(audit) == [expected, expected] @@ -1899,6 +1961,201 @@ def ad_reconcile_aborted( await store.close() +def _no_such_base() -> LdapUnansweredError: + """What ``_search`` raises when the user search base was deleted: engine text, a code and its + name, never the directory's diagnostic message.""" + return LdapUnansweredError( + "AD ended the user search with result code 32 (noSuchObject), which is not an answer", + code=32, + code_name="noSuchObject", + ) + + +#: The two unjudged outcomes that count over the probes asked besides the referral: what makes +#: ``alice``'s probe one, its audit action, the row field holding its count, the plan field, the +#: latched message's property and the alert's reason. +_COUNTED_OVER_ASKED = [ + pytest.param( + lambda: KeyError("userAccountControl"), + "auth.ad_reconcile_faulted", + "faulted", + "directory_reconcile_fault", + reconcile.PROBE_FAULT_ABORT, + id="probe-fault", + ), + pytest.param( + _no_such_base, + "auth.ad_reconcile_misconfigured", + "misconfigured", + "directory_reconcile_misconfigured", + reconcile.MISCONFIGURED_ABORT, + id="configuration-refusal", + ), +] + + +@pytest.mark.parametrize(("raised", "action", "field", "latch", "reason"), _COUNTED_OVER_ASKED) +async def test_a_fault_or_refusal_is_counted_over_the_probes_asked( + raised: Any, action: str, field: str, latch: str, reason: str +) -> None: + """The twin of the referral's test above (BACKLOG #3170, Manager ruling: one event, one count). + The fault's row, message and alert count the probes asked, so the id-less row is left out, and + the row names the field ``asked``. The configuration refusal (BACKLOG #3010) does the same. + + RED before D3b: the fault's row carried ``probed`` and its alert counted the id-less row.""" + from messagefoundry.api.app import _alert_reconcile_plan + from messagefoundry.pipeline.alerts import LoggingAlertSink + + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap( + { + "alice": _principal("alice"), + "bob": _principal("bob"), + "jdoe": replace(_principal("jdoe"), directory_object_id=None), + } + ) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + for name in ("alice", "bob", "jdoe"): + assert await _signed_in_ad_user(service, store, name) is not None + ldap.broken["alice"] = raised() + + plan = await service.reconcile_directory_sessions() + + assert (plan.probed, plan.asked, len(getattr(plan, field))) == (3, 2, 1) + [row] = [json.loads(a["detail"]) for a in await store.list_audit() if a["action"] == action] + assert row["asked"] == 2 and "probed" not in row + message = getattr(service, latch) + assert message is not None and "1 of 2 signed-in" in message + + counts: list[tuple[str, int]] = [] + + class _Sink(LoggingAlertSink): + def ad_reconcile_aborted( + self, name: str, *, reason: str, probed: int, detail: str + ) -> None: + counts.append((reason, probed)) + + _alert_reconcile_plan(plan, service, _Sink()) + assert counts == [(reason, 2)] + # The breaker's page keeps its judged denominator, which counts the id-less row. + counts.clear() + _alert_reconcile_plan(replace(plan, aborted="mass_revoke_breaker"), service, _Sink()) + assert counts == [("mass_revoke_breaker", 3), (reason, 2)] + finally: + await store.close() + + +@pytest.mark.parametrize(("raised", "action", "field", "latch", "reason"), _COUNTED_OVER_ASKED) +async def test_a_pass_of_faults_or_refusals_only_strikes_an_id_less_row_and_never_revokes_it( + raised: Any, action: str, field: str, latch: str, reason: str +) -> None: + """BACKLOG #3170 item 3 at the service, for a pass that judged nothing the directory answered. + The id-less row is struck on every such pass, past the strike threshold, and its session is + never revoked: only a pass that is not aborted can revoke it. Then one such pass ends, and the + row is revoked on the next clean pass, which proves the strikes were counted.""" + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap( + { + "alice": _principal("alice"), + "jdoe": replace(_principal("jdoe"), directory_object_id=None), + } + ) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + token = await _signed_in_ad_user(service, store, "jdoe") + assert token is not None and await _signed_in_ad_user(service, store, "alice") is not None + jdoe = await store.get_user_by_username("jdoe") + assert jdoe is not None + ldap.broken["alice"] = raised() + + for strike in (1, 2, 3): # the threshold is 2 + plan = await service.reconcile_directory_sessions() + assert plan.aborted == reason and plan.judged_nothing + assert plan.revocations == () and plan.strikes == {jdoe.id: strike} + assert await service.identity_for_token(token) is not None, "an aborted pass revoked" + audit = await store.list_audit() + assert not any(a["action"] == "auth.ad_session_revoked" for a in audit) + + del ldap.broken["alice"] # THE CONTROL: the first pass that is not aborted revokes it + plan = await service.reconcile_directory_sessions() + assert [(r.user_id, r.reason) for r in plan.revocations] == [ + (jdoe.id, "directory_object_id_missing") + ] + finally: + await store.close() + + +async def test_a_deleted_search_base_aborts_alerts_and_never_revokes( + caplog: pytest.LogCaptureFixture, +) -> None: + """BACKLOG #3010 limb 5. A deleted or moved user search base answers noSuchObject for every + account on every pass. Before #3010 it read "not found", struck everyone and tripped the + breaker; with #3010 alone it read as an outage, which pages nobody. Now each pass aborts as + ``directory_misconfigured``: one ERROR naming the code and no account, a latched message + naming the settings, a misconfigured row, and no skipped or aborted row. One refusal beside a + healthy account is recorded the same way; a pass with none releases the message. + + RED before D3b: the pass is ``directory_unavailable`` and nothing latches.""" + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap({"jdoe": _principal("jdoe"), "asmith": _principal("asmith")}) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + tokens = [await _signed_in_ad_user(service, store, u) for u in ("jdoe", "asmith")] + for name in ("jdoe", "asmith"): + ldap.broken[name] = _no_such_base() + + for _ in range(3): # past the strike threshold + caplog.clear() + with caplog.at_level(logging.WARNING, logger="messagefoundry.auth.service"): + plan = await service.reconcile_directory_sessions() + assert plan.aborted == reconcile.MISCONFIGURED_ABORT and plan.directory_misconfigured + assert not plan.directory_outage and plan.judged_nothing + assert len(plan.misconfigured) == 2 and plan.revocations == () + assert not plan.breaker_clear and not plan.hold_clear and not plan.referral_clear + errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + assert len(errors) == 1 and "noSuchObject" in errors[0] + assert "jdoe" not in errors[0] and "asmith" not in errors[0] + for token in tokens: + assert token is not None and await service.identity_for_token(token) is not None + message = service.directory_reconcile_misconfigured + assert message is not None and "revoked nothing" in message + assert "ad_user_search_base" in message[:200] + audit = await store.list_audit() + rows = [ + json.loads(a["detail"]) + for a in audit + if a["action"] == "auth.ad_reconcile_misconfigured" + ] + expected = { + "reason": "directory_misconfigured", + "misconfigured": 2, + "unavailable": 0, + "asked": 2, + "aborted": "directory_misconfigured", + } + assert rows == [expected] * 3 + assert not any( + a["action"] in ("auth.ad_reconcile_skipped", "auth.ad_reconcile_aborted") for a in audit + ) + + del ldap.broken["asmith"] # one refusal beside a healthy account + plan = await service.reconcile_directory_sessions() + assert plan.aborted is None and plan.misconfigured != () + assert service.directory_reconcile_misconfigured is not None + assert "judged as usual" in service.directory_reconcile_misconfigured + + del ldap.broken["jdoe"] # the base is restored + plan = await service.reconcile_directory_sessions() + assert plan.aborted is None and plan.misconfigured == () + assert service.directory_reconcile_misconfigured is None + finally: + await store.close() + + async def test_a_breaker_trip_does_not_mark_an_id_less_row_or_forfeit_its_clear() -> None: """BACKLOG #2434, the breaker's half of the referral test above. A trip marks every candidate unconfirmed until a pass reads it PRESENT, and revoking a marked account forfeits the clear. An @@ -2602,8 +2859,9 @@ class _CapturingNotifier: def __init__(self) -> None: self.sent: list[Any] = [] - async def notify(self, event: Any) -> None: + async def notify(self, event: Any) -> bool: self.sent.append(event) + return True async def test_a_rename_and_a_role_change_in_one_pass_record_one_consistent_name() -> None: diff --git a/tests/test_ad_user_account_control.py b/tests/test_ad_user_account_control.py index 9558e2b86..9b3afa387 100644 --- a/tests/test_ad_user_account_control.py +++ b/tests/test_ad_user_account_control.py @@ -39,6 +39,7 @@ from messagefoundry.api import app as app_module from messagefoundry.api import create_managed_app from messagefoundry.api.app import ( + _MISCONFIGURED_ALERT_SOURCE, _PROBE_FAULT_ALERT_SOURCE, _RECONCILE_CLEAR_RESOLVES, _REFERRAL_ALERT_SOURCE, @@ -52,6 +53,7 @@ from messagefoundry.auth.permissions import Role from messagefoundry.auth.reconcile import ( HOLD_REASON, + MISCONFIGURED_ABORT, PROBE_FAULT_ABORT, REFERRAL_ABORT, ProbeOutcome, @@ -959,6 +961,7 @@ def ad_reconcile_hold_released(self, name: str) -> None: directory_reconcile_hold=None, directory_reconcile_referral=None, directory_reconcile_fault=None, + directory_reconcile_misconfigured=None, ) #: The referral's own source label (BACKLOG #2538), apart from the breaker's. @@ -1097,6 +1100,74 @@ def test_a_probe_fault_pages_under_its_own_source( assert faulted and all(e[2]["reason"] == PROBE_FAULT_ABORT for e in faulted) +_REFUSAL_PAGE = ("ad_reconcile_aborted", _MISCONFIGURED_ALERT_SOURCE) + + +@pytest.mark.parametrize( + ("plan", "expected"), + [ + ( + ReconcilePlan( + probed=3, + asked=3, + misconfigured=("a",), + revocations=(SessionRevocation("b", "bwong", reason="directory_absent"),), + ), + [("ad_session_revoked", "bwong"), _REFUSAL_PAGE], + ), + ( + # A pass of refusals only: the refusal's page alone, never the breaker's instance. + ReconcilePlan(probed=2, asked=2, misconfigured=("a", "b"), aborted=MISCONFIGURED_ABORT), + [_REFUSAL_PAGE], + ), + ( + # A pass of referrals and refusals only: one page on each of their instances. + ReconcilePlan( + probed=2, asked=2, referred=("a",), misconfigured=("b",), aborted=REFERRAL_ABORT + ), + [("ad_reconcile_aborted", _REFERRAL_SOURCE), _REFUSAL_PAGE], + ), + ( + ReconcilePlan( + probed=2, asked=2, misconfigured=("a",), faulted=("b",), aborted=MISCONFIGURED_ABORT + ), + [_REFUSAL_PAGE, _FAULT_PAGE], + ), + ( + ReconcilePlan(probed=12, asked=12, misconfigured=("a",), aborted="mass_revoke_breaker"), + [("ad_reconcile_aborted", "directory-reconciler"), _REFUSAL_PAGE], + ), + ( + # THE CONTROL: a plain outage pages nothing. + ReconcilePlan(probed=2, asked=2, unavailable=2, aborted="directory_unavailable"), + [], + ), + ], + ids=[ + "beside-a-revocation", + "refusals-only", + "beside-a-referral", + "beside-a-fault", + "beside-a-trip", + "outage", + ], +) +def test_a_configuration_refusal_pages_under_its_own_source( + plan: ReconcilePlan, expected: list[tuple[str, str]] +) -> None: + """BACKLOG #3010 limb 5. A deleted or moved search base read as an outage with #3010, which + pages nobody. Every pass with a configuration refusal pages under its own source, counted + over the probes asked, so an acknowledged refusal never silences a breaker trip or a referral. + + RED before D3b: a pass of refusals only pages nothing.""" + sink = _InverseSink() + _alert_reconcile_plan(plan, _AUTH, sink) # type: ignore[arg-type] + assert [(e[0], e[1]) for e in sink.events] == expected + for event in sink.events: + if (event[0], event[1]) == _REFUSAL_PAGE: + assert event[2]["reason"] == MISCONFIGURED_ABORT and event[2]["probed"] == plan.asked + + def test_the_inverses_resolve_their_alerts_and_are_not_rule_targetable() -> None: assert _AUTO_RESOLVE["ad_reconcile_breaker_cleared"] == "ad_reconcile_aborted" assert _AUTO_RESOLVE["ad_reconcile_hold_released"] == "ad_reconcile_held" diff --git a/tests/test_admin_new_ip.py b/tests/test_admin_new_ip.py index 36ad5eaa8..239f07e10 100644 --- a/tests/test_admin_new_ip.py +++ b/tests/test_admin_new_ip.py @@ -65,8 +65,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _enabled_admin(service: AuthService, *, client: str) -> tuple[str, Identity]: diff --git a/tests/test_admin_notify_email_update.py b/tests/test_admin_notify_email_update.py index 3cedf9312..73da029e5 100644 --- a/tests/test_admin_notify_email_update.py +++ b/tests/test_admin_notify_email_update.py @@ -48,8 +48,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _account(store: MessageStore, *, notify: str | None, profile: str | None) -> None: diff --git a/tests/test_audit_actor_index.py b/tests/test_audit_actor_index.py new file mode 100644 index 000000000..623bec3e5 --- /dev/null +++ b/tests/test_audit_actor_index.py @@ -0,0 +1,250 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""Vault BACKLOG #3120, item (d) and review follow-up 5: the audit actor indexes. + +The lock notice's 24-hour throttle reads one account's ``auth.lock_notice`` rows through +``list_audit(actor=, action=, since=, limit=50)``. With only ``ix_audit_ts`` and +``ix_audit_action`` that read walked every recent row of the action, for every account. +``ix_audit_actor`` (actor, action, id) lets it seek the account's own rows, newest first. + +That index alone regressed the reads filtered by actor WITHOUT an action equality (``GET +/audit?actor=``, its export pages, ``/me/security-events``): SQLite sought the actor on it and then +sorted every one of the actor's rows to return the newest few. ``ix_audit_actor_id`` (actor, id) +serves those reads in id order. On Postgres both key a prefix of the actor, which is unbounded TEXT +there. Only the SQLite plans are measured here; on the server backends this file checks that the +indexes exist, not that the planner picks them. + +Backend-parametrized like ``test_fifo_index_migration``: the SQLite case runs everywhere; the SQL +Server and Postgres cases run only when their ``MEFOR_TEST_*`` env is set, which the hosted service- +container legs do. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import os +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from pathlib import Path +from typing import Any + +import pytest + +from messagefoundry.store import MessageStore +from messagefoundry.store.audit_exclusion import AuditExclusion + +_SQLSERVER_ON = bool(os.getenv("MEFOR_TEST_SQLSERVER")) +_POSTGRES_ON = bool(os.getenv("MEFOR_TEST_POSTGRES")) + +_LOCK_NOTICE = "auth.lock_notice" + + +async def _open(backend: str, path: Path) -> Any: + if backend == "sqlite": + return await MessageStore.open(path) + from messagefoundry.config.settings import load_settings + + settings = load_settings(environ=os.environ).store + if backend == "postgres": + from messagefoundry.store.postgres import PostgresStore + + return await PostgresStore.open(settings) + from messagefoundry.store.sqlserver import SqlServerStore + + return await SqlServerStore.open(settings) + + +@pytest.fixture(params=["sqlite", "sqlserver", "postgres"]) +async def store(request: pytest.FixtureRequest, tmp_path: Path) -> AsyncIterator[Any]: + backend = request.param + if backend == "sqlserver" and not _SQLSERVER_ON: + pytest.skip("set MEFOR_TEST_SQLSERVER=1 (+ MEFOR_STORE_* env) to run the SQL Server case") + if backend == "postgres" and not _POSTGRES_ON: + pytest.skip("set MEFOR_TEST_POSTGRES=1 (+ MEFOR_STORE_* env) to run the Postgres case") + s: Any = await _open(backend, tmp_path / "actor_idx.db") + s._test_backend = backend + try: + yield s + finally: + await s.close() + + +async def _index_columns(store: Any, name: str) -> list[str]: + """The key columns of the ``audit_log`` index ``name``, in key order; empty when it is absent.""" + backend = store._test_backend + if backend == "sqlite": + async with store._read() as db: + cur = await db.execute(f"PRAGMA index_info({name})") + return [r[2] for r in sorted(await cur.fetchall(), key=lambda r: r[0])] + if backend == "postgres": + # pg_get_indexdef per key position, so an expression key reads as its expression. The + # store's own schema only, so a same-named index in another schema cannot answer for it. + # Compared as a name, not cast to regnamespace, which would case-fold a mixed-case schema. + rows = await store._fetchall( + "SELECT pg_get_indexdef(c.oid, k.ord, true) AS col FROM pg_class c" + " JOIN pg_index i ON i.indexrelid = c.oid" + " JOIN pg_namespace n ON n.oid = c.relnamespace" + " CROSS JOIN LATERAL generate_series(1, i.indnkeyatts::int) AS k(ord)" + " WHERE c.relname = $1 AND n.nspname = current_schema()" + " ORDER BY k.ord", + name, + ) + return [str(r["col"]).replace('"', "").replace(" ", "") for r in rows] + rows = await store._fetchall( # sqlserver + "SELECT c.name FROM sys.indexes i" + " JOIN sys.index_columns ic ON ic.object_id=i.object_id AND ic.index_id=i.index_id" + " JOIN sys.columns c ON c.object_id=i.object_id AND c.column_id=ic.column_id" + " WHERE i.object_id=OBJECT_ID('audit_log') AND i.name=? AND ic.key_ordinal > 0" + " ORDER BY ic.key_ordinal", + (name,), + ) + return [r["name"] for r in rows] + + +async def test_the_actor_indexes_exist_on_every_backend(store: Any) -> None: + # Postgres keys a prefix of the unbounded TEXT actor; see the next test. + actor = "left(actor,200)" if store._test_backend == "postgres" else "actor" + assert await _index_columns(store, "ix_audit_actor") == [actor, "action", "id"] + assert await _index_columns(store, "ix_audit_actor_id") == [actor, "id"] + # The control: the same probe reads the index that was already there, so an empty list above + # would be the index missing, not the probe failing. + assert await _index_columns(store, "ix_audit_action") == ["action", "id"] + + +async def test_an_overlong_actor_is_still_audited_and_found(store: Any) -> None: + """A btree entry over about 2.7 KB fails its INSERT on Postgres, and ``actor`` is unbounded + there. An index on the whole value would turn an overlong typed username into a failed audit + write. The Postgres indexes key a prefix, and ``list_audit`` still matches the whole value, + with or without an action filter. + + SQL Server is left out: its ``actor`` column is NVARCHAR(256), which refuses such a row at the + column whatever the index, so the case is not this change's to pin there.""" + if store._test_backend == "sqlserver": + pytest.skip("SQL Server's actor column is NVARCHAR(256)") + # Random hex, which pglz cannot shrink, so compressing the index entry cannot rescue it. + long_actor = "".join(hashlib.sha256(str(i).encode()).hexdigest() for i in range(125)) # 8000 + near = long_actor[:200] + "-another" # same prefix, another actor + await store.record_audit(_LOCK_NOTICE, actor=long_actor, detail='{"lock": "sign_in"}') + await store.record_audit(_LOCK_NOTICE, actor=near, detail='{"lock": "second_step"}') + rows = await store.list_audit(actor=long_actor, action=_LOCK_NOTICE, since=0.0, limit=50) + assert [r["detail"] for r in rows] == ['{"lock": "sign_in"}'] + rows = await store.list_audit(actor=long_actor, since=0.0, limit=50) + assert [r["detail"] for r in rows] == ['{"lock": "sign_in"}'] + + +async def test_security_events_tell_apart_two_names_sharing_the_indexed_prefix(store: Any) -> None: + """``/me/security-events`` filters on the Postgres actor prefix too, and only the whole-value + term keeps one account from reading another's events when both names share those 200 + characters. 250 characters fits every backend's username and actor columns. Random names, so a + shared server database run twice does not meet the first run's rows.""" + stem = "".join(hashlib.sha256(os.urandom(16)).hexdigest() for _ in range(4))[:200] + mine, theirs = stem + "a" * 50, stem + "b" * 50 + for n, name in enumerate((mine, theirs)): + await store.create_user( + user_id=f"u-{stem[:16]}-{n}", + username=name, + auth_provider="local", + password_generated=False, + now=1.0, + ) + await store.record_audit("auth.login_success", actor=mine, detail="mine") + await store.record_audit("auth.login_success", actor=theirs, detail="theirs") + rows = await store.security_events_for_user(mine) + assert [r["detail"] for r in rows] == ["mine"] + assert await store.count_security_events_for_user(mine) == 1 + + +async def _plan( + store: MessageStore, + monkeypatch: pytest.MonkeyPatch, + call: Callable[[MessageStore], Awaitable[object]], +) -> str: + """The SQLite plan of the one statement ``call`` runs on the store's read connection. + + The statement and its parameters are captured off the read connection as the store method + really runs them, not rebuilt here, so the SQL cannot drift from the method.""" + real_read = store._read + seen: list[tuple[str, Sequence[Any]]] = [] + + class _Spy: + def __init__(self, db: Any) -> None: + self._db = db + + async def execute(self, sql: str, params: Sequence[Any] = ()) -> Any: + seen.append((sql, params)) + return await self._db.execute(sql, params) + + @contextlib.asynccontextmanager + async def spy_read() -> AsyncIterator[_Spy]: + async with real_read() as db: + yield _Spy(db) + + with monkeypatch.context() as m: + m.setattr(store, "_read", spy_read) + await call(store) + [(sql, params)] = seen + async with real_read() as db: + cur = await db.execute(f"EXPLAIN QUERY PLAN {sql}", params) + return " | ".join(str(r[-1]) for r in await cur.fetchall()) + + +async def test_the_throttle_read_seeks_the_actor_index_on_sqlite( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The plan of the statement ``list_audit`` runs for the throttle seeks ``ix_audit_actor`` on + both the actor and the action, not ``ix_audit_actor_id`` on the actor alone. + + The filters are the ones ``AuthService._lock_notice_held_back`` passes today, written out here. + The control is an action-only read, whose plan names ``ix_audit_action``, so the instrument is + known to report the index it uses.""" + store = await MessageStore.open(tmp_path / "plan.db") + try: + throttle = await _plan( + store, + monkeypatch, + lambda s: s.list_audit(actor="owner", action=_LOCK_NOTICE, since=0.0, limit=50), + ) + control = await _plan(store, monkeypatch, lambda s: s.list_audit(action=_LOCK_NOTICE)) + finally: + await store.close() + assert "ix_audit_actor (actor=? AND action=?)" in throttle, throttle + assert "TEMP B-TREE" not in throttle, throttle + assert "ix_audit_action" in control, control + + +# At least these reads filter by actor without an action equality: the audit page by offset and its +# capped total, the export's keyset cursor, a time-bounded read, the page a reader without +# users:manage gets (an exclusion), and the self-service security events. +_HIDDEN = AuditExclusion( + actions=frozenset({"auth.hidden"}), rows=frozenset({("auth.login_failed", "hidden")}) +) +_ACTOR_ONLY_READS: dict[str, Callable[[MessageStore], Awaitable[object]]] = { + "list_audit": lambda s: s.list_audit(actor="owner", limit=50), + "list_audit_offset": lambda s: s.list_audit(actor="owner", limit=50, offset=500), + "list_audit_before_id": lambda s: s.list_audit(actor="owner", limit=50, before_id=1000), + "list_audit_excluded": lambda s: s.list_audit(actor="owner", limit=50, exclude=_HIDDEN), + "list_audit_since": lambda s: s.list_audit(actor="owner", since=0.0, until=9e9, limit=50), + "count_audit_capped": lambda s: s.count_audit(actor="owner", limit=1000), + "security_events": lambda s: s.security_events_for_user("owner", limit=100, until=9e9), +} + + +@pytest.mark.parametrize("read", sorted(_ACTOR_ONLY_READS)) +async def test_an_actor_only_read_walks_the_actor_in_id_order_on_sqlite( + read: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With ``ix_audit_actor`` alone, SQLite sought the actor on it and then sorted all of the + actor's rows into a temporary B-tree to read the newest few. Measured with 300k of 600k rows + under one actor: a first page took 23 ms against 0.06 ms before that index, and a page at + offset 100000 took 233 ms against 8.6 ms. ``(actor, id)`` hands the rows over already in ``id`` + order: 0.05 ms and 2.9 ms. + + An empty database plans like a full one here, because SQLite has no statistics until ANALYZE + and the store never runs it.""" + store = await MessageStore.open(tmp_path / "plan.db") + try: + plan = await _plan(store, monkeypatch, _ACTOR_ONLY_READS[read]) + finally: + await store.close() + assert "ix_audit_actor_id (actor=?" in plan, plan + assert "TEMP B-TREE" not in plan, plan diff --git a/tests/test_auth_hardening.py b/tests/test_auth_hardening.py index 435bbd18e..aa83fcba7 100644 --- a/tests/test_auth_hardening.py +++ b/tests/test_auth_hardening.py @@ -899,11 +899,17 @@ async def _run_reaper( class _Store: async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> None: purged.append(now) if inner is not None: - await inner.purge_expired_sessions(now=now, idle_seconds=idle_seconds) + await inner.purge_expired_sessions( + now=now, idle_seconds=idle_seconds, lineage_seconds=lineage_seconds + ) def _wall() -> float: try: diff --git a/tests/test_auth_oidc_service.py b/tests/test_auth_oidc_service.py index 2c61b681e..9fe01f4d2 100644 --- a/tests/test_auth_oidc_service.py +++ b/tests/test_auth_oidc_service.py @@ -856,8 +856,9 @@ class _CapturingNotifier: def __init__(self) -> None: self.events: list[Any] = [] - async def notify(self, event: Any) -> None: + async def notify(self, event: Any) -> bool: self.events.append(event) + return True async def test_the_admin_bind_audits_and_notifies_and_a_login_never_does( diff --git a/tests/test_auth_service.py b/tests/test_auth_service.py index 5d94184cc..8da1b7844 100644 --- a/tests/test_auth_service.py +++ b/tests/test_auth_service.py @@ -50,8 +50,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _store() -> MessageStore: @@ -450,9 +451,9 @@ async def _local_user(store: MessageStore, *, email: str = "bob@example.org") -> async def test_notifier_fires_once_on_account_lockout() -> None: store = await _store() + notifier = _FakeNotifier() + service = AuthService(store, AuthSettings(lockout_threshold=3), security_notifier=notifier) try: - notifier = _FakeNotifier() - service = AuthService(store, AuthSettings(lockout_threshold=3), security_notifier=notifier) await _local_user(store) for _ in range(3): await service.login("bob", "wrong", client="10.0.0.9") @@ -464,6 +465,8 @@ async def test_notifier_fires_once_on_account_lockout() -> None: assert locked[0].email == "bob@example.org" assert locked[0].client_ip == "10.0.0.9" finally: + # Again here, so a failure above cannot leave the notice running (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() @@ -601,7 +604,7 @@ async def test_notifier_absent_does_not_break_auth() -> None: class _BoomNotifier: """A notifier whose notify() always raises — exercises AuthService's best-effort guard.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise RuntimeError("notifier down") @@ -631,7 +634,7 @@ async def test_notifier_failure_is_isolated_from_the_auth_op() -> None: class _AddressQuotingNotifier: """A notifier that raises with the event's address and detail in its message.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise RuntimeError(f"cannot mail {event.email}: {event.detail}") diff --git a/tests/test_combined_sign_in_audit_oracle.py b/tests/test_combined_sign_in_audit_oracle.py index 721ef2a49..27ea3f862 100644 --- a/tests/test_combined_sign_in_audit_oracle.py +++ b/tests/test_combined_sign_in_audit_oracle.py @@ -245,8 +245,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[Any] = [] - async def notify(self, event: Any) -> None: + async def notify(self, event: Any) -> bool: self.events.append(event) + return True # --- The lock-event limb, closed by owner ruling 2026-09-28 -------------------------------------- diff --git a/tests/test_initial_credential_reminder.py b/tests/test_initial_credential_reminder.py index 645abb23c..272a6838f 100644 --- a/tests/test_initial_credential_reminder.py +++ b/tests/test_initial_credential_reminder.py @@ -234,8 +234,9 @@ class _RecordingNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def reminders(self, holder: str) -> list[tuple[str, str, str | None, dict[str, Any]]]: """The reminders about ``holder``'s credential: its own, and its issuer's naming it. Other diff --git a/tests/test_ldap_decode_boundary.py b/tests/test_ldap_decode_boundary.py index 8c80cb213..bf7756a22 100644 --- a/tests/test_ldap_decode_boundary.py +++ b/tests/test_ldap_decode_boundary.py @@ -25,6 +25,7 @@ import ast import contextlib import inspect +import time from collections.abc import AsyncIterator, Callable from types import SimpleNamespace from typing import Any @@ -241,12 +242,9 @@ def _assert_referred_bind(exc: BaseException, operation: str) -> None: def test_a_service_bind_answered_with_a_referral_is_a_referral() -> None: - """Before D3b it read as an undecodable response, an outage the reconciler pages nobody for. - THE PREMISE first: the bare ldap3 bind raises ``KeyError(3)``. The socket is still closed.""" - with _scripted_dc(lambda _raw: [], bind_reply=lambda _raw: (_REFERRAL_BIND, b"")) as ( - dc, - authenticator, - ): + """Before D3b it read as an undecodable response, not as a referral. THE PREMISE first: the + bare ldap3 bind raises ``KeyError(3)``. The socket is still closed.""" + with _bad_bind_dc(_REFERRAL_BIND) as (dc, authenticator): with pytest.raises(KeyError) as bare: _bare_bind(dc.port) assert bare.value.args == (3,) @@ -278,21 +276,10 @@ def bind(raw: bytes) -> _Reply | None: async def test_the_reconciler_reads_a_referred_service_bind_as_referred() -> None: - """So the probe reaches the referral's alert path (BACKLOG #2538), not the outage's.""" - with _scripted_dc(lambda _raw: [], bind_reply=lambda _raw: (_REFERRAL_BIND, b"")) as ( - dc, - authenticator, - ): - store = await MessageStore.open(":memory:") - try: - settings = _ad_settings( - ad_server=f"ldap://127.0.0.1:{dc.port}", ad_allow_insecure_ldap=True - ) - service = AuthService(store, settings, ldap=authenticator) - user = SimpleNamespace(id="u-1", username="alice", directory_object_id=None) - probe = await service._probe_principal(user) # type: ignore[arg-type] - finally: - await store.close() + """So the probe reaches the referral's alert path (BACKLOG #2538), not the outage's or the + probe fault's, and its detail names no referred host.""" + with _bad_bind_dc(_REFERRAL_BIND) as (dc, authenticator): + probe = await _probe_through_the_reconciler(authenticator, dc.port) assert probe.outcome is ProbeOutcome.REFERRED assert probe.detail is not None and _REFERRED not in probe.detail @@ -317,6 +304,60 @@ def test_a_referral_in_a_who_am_i_response_is_still_undecodable() -> None: assert "Who am I with a response ldap3 could not decode (KeyError)" in reading.whoami_error +# --- the service connection is released by every lookup (D3b) ------------------------------------ + + +def _groups_answered(raw: bytes) -> list[_Reply]: + """The user search finds alice; the nested group search ends with success and no group.""" + return [_done(RESULT_SUCCESS)] if _IN_CHAIN in raw else _alice(raw) + + +def _group_search_busy(raw: bytes) -> list[_Reply]: + return [_done(51)] if _IN_CHAIN in raw else _alice(raw) + + +def _group_page_undecodable(raw: bytes) -> list[_Reply]: + no_value = _tlv(0x30, _tlv(0x04, _OID)) # ldap3 raises TypeError on it + return [_bad_page(no_value)] if _IN_CHAIN in raw else _alice(raw) + + +def _probe(name: str) -> Callable[[Any], object]: + return lambda authenticator: authenticator.probe_principal(name) + + +def _sign_in(name: str) -> Callable[[Any], object]: + return lambda authenticator: authenticator.authenticate(name, "synthetic-user-pw") + + +@pytest.mark.parametrize( + ("answer", "call", "sessions"), + [ + pytest.param(_groups_answered, _probe("alice"), 1, id="probe-present"), + pytest.param(_groups_answered, _probe("nobody"), 1, id="probe-absent"), + pytest.param(_group_search_busy, _probe("alice"), 1, id="probe-group-search-busy"), + pytest.param(_group_page_undecodable, _probe("alice"), 1, id="probe-undecodable-page"), + pytest.param(_groups_answered, _sign_in("alice"), 2, id="sign-in-present"), + pytest.param(_groups_answered, _sign_in("nobody"), 2, id="sign-in-absent-decoy"), + pytest.param(_group_search_busy, _sign_in("alice"), 2, id="sign-in-group-search-busy"), + ], +) +def test_every_directory_session_a_lookup_opens_is_ended( + answer: Callable[[bytes], list[_Reply]], call: Callable[[Any], object], sessions: int +) -> None: + """D3b, a finding of BACKLOG #3010's review. ``authenticate`` and ``probe_principal`` held the + service connection in ldap3's own ``with``, which never unbinds a connection bound on entry, + so every lookup left one directory session open until the connection was collected. Counted + at the loopback server, by the unbind requests it receives, whatever the lookup answered.""" + with _scripted_dc(answer) as (dc, authenticator): + with contextlib.suppress(LdapError): + call(authenticator) + assert dc.opened == sessions, "the lookup did not open the sessions this case names" + deadline = time.monotonic() + _CLOSE_WAIT + while dc.unbound < dc.opened and time.monotonic() < deadline: + time.sleep(0.01) + assert dc.unbound == dc.opened, f"{dc.opened} session(s) opened, {dc.unbound} ended" + + # --- the nested group search: a malformed paged results control --------------------------------- _OID = ldap_module._PAGED_RESULTS_OID.encode() @@ -667,6 +708,22 @@ def test_an_unbind_error_never_replaces_the_refusal() -> None: pass +def test_a_probe_answered_before_the_unbind_fails_keeps_its_answer() -> None: + """The service connection's release, end to end (``_service_session``, BACKLOG #3017): unlike + ``_released`` above after a clean block, an unbind whose send fails is suppressed, and the probe + still answers.""" + with _scripted_dc(_groups_answered) as (_dc, authenticator): + real = ldap3.Connection.unbind + + def failing_unbind(self: Any) -> Any: + real(self) + raise ldap3.core.exceptions.LDAPSocketSendError("synthetic") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(ldap3.Connection, "unbind", failing_unbind) + assert authenticator.probe_principal("alice").principal is not None + + def test_the_refusal_warns_once_per_operation_and_fault(caplog: pytest.LogCaptureFixture) -> None: with _scripted_dc(lambda _raw: [_done(4242)]) as (_dc, authenticator): for _ in range(3): diff --git a/tests/test_ldap_search_result_codes.py b/tests/test_ldap_search_result_codes.py index 710538194..112574a95 100644 --- a/tests/test_ldap_search_result_codes.py +++ b/tests/test_ldap_search_result_codes.py @@ -65,11 +65,12 @@ LdapUnansweredError, ) from messagefoundry.auth.permissions import Role -from messagefoundry.auth.reconcile import ProbeOutcome +from messagefoundry.auth.reconcile import MISCONFIGURED_ABORT, ProbeOutcome from messagefoundry.auth.service import AuthService from messagefoundry.config.settings import AuthSettings from messagefoundry.store.store import MessageStore from tests.test_ad_user_account_control import _alive, _audited, _pass, _signed_in_estate +from tests.test_approval_requester_recheck import _Sink from tests.test_ldap_referrals import ( _BIND_REQUEST, _BIND_RESPONSE, @@ -311,27 +312,28 @@ def test_a_group_search_the_directory_did_not_finish_is_an_outage_not_fewer_grou @pytest.mark.parametrize( - ("code", "partial"), + ("code", "partial", "outcome"), [ - (RESULT_BUSY, False), - (RESULT_ADMIN_LIMIT_EXCEEDED, False), - (RESULT_NO_SUCH_OBJECT, False), - (RESULT_SIZE_LIMIT_EXCEEDED, True), + (RESULT_BUSY, False, ProbeOutcome.UNAVAILABLE), + (RESULT_ADMIN_LIMIT_EXCEEDED, False, ProbeOutcome.UNAVAILABLE), + (RESULT_NO_SUCH_OBJECT, False, ProbeOutcome.MISCONFIGURED), + (RESULT_SIZE_LIMIT_EXCEEDED, True, ProbeOutcome.UNAVAILABLE), ], ids=["busy", "adminLimitExceeded", "noSuchObject", "sizeLimit-partial"], ) async def test_a_directory_that_did_not_answer_never_revokes_a_present_account( - monkeypatch: pytest.MonkeyPatch, code: int, partial: bool + monkeypatch: pytest.MonkeyPatch, code: int, partial: bool, outcome: ProbeOutcome ) -> None: """The row's defect, end to end. One signed-in account, under the breaker's floor, whose user search the directory answers with ``code`` on every pass. Before the fix the user search read - ABSENT and the account would be revoked at the strike threshold. Now each probe is unavailable - and the session survives. The group search has its own test below, which also reads roles.""" + ABSENT and the account would be revoked at the strike threshold. Now each probe is unavailable, + or misconfigured for a code that points at the configuration, and the session survives. The + group search has its own test below, which also reads roles.""" async with _signed_in_estate(monkeypatch, ["jdoe"]) as (_directory, service, store, tokens): _answer_with(monkeypatch, code, when=_user_search, keep_entries=partial) user = await store.get_user_by_username("jdoe") assert user is not None - assert (await service._probe_principal(user)).outcome is ProbeOutcome.UNAVAILABLE + assert (await service._probe_principal(user)).outcome is outcome for _ in range(5): # far past ad_session_recheck_strikes assert (await _pass(service)).revocations == () assert await _alive(service, tokens) == {"jdoe"} @@ -353,6 +355,64 @@ async def test_an_account_that_left_the_directory_is_still_revoked( assert await _alive(service, tokens) == set() +@pytest.mark.parametrize( + ("code", "paged"), + [ + (RESULT_NO_SUCH_OBJECT, True), + (RESULT_INVALID_DN_SYNTAX, True), + (RESULT_INSUFFICIENT_ACCESS_RIGHTS, True), + # THE CONTROLS: a transient code is a plain outage, which pages nobody. + (RESULT_BUSY, False), + (RESULT_UNAVAILABLE, False), + ], + ids=[ + "32-noSuchObject", + "34-invalidDNSyntax", + "50-insufficientAccess", + "51-busy", + "52-unavailable", + ], +) +async def test_a_refused_user_search_base_above_the_breaker_floor_pages_and_never_revokes( + monkeypatch: pytest.MonkeyPatch, code: int, paged: bool +) -> None: + """D3b, #3010 limb 5, through the real authenticator. Eight signed-in accounts, above the + breaker's floor, whose user search base the directory refuses on every pass. Before #3010 a + deleted base read "not found", struck all eight and tripped the breaker; with #3010 alone it + read as an outage that paged nobody. A configuration code now aborts every pass as + ``directory_misconfigured`` and pages under its own source, counted over the probes asked. It + never revokes, and it is never the breaker's trip. + + RED before D3b for the three configuration codes: each pass is a plain outage.""" + names = [f"user{i:02d}" for i in range(8)] + async with _signed_in_estate(monkeypatch, names) as (_directory, service, store, tokens): + _answer_with(monkeypatch, code, when=_user_search) + sink = _Sink() + for _ in range(4): # far past ad_session_recheck_strikes + plan = await _pass(service, sink) + assert plan.revocations == () + if paged: + assert plan.aborted == MISCONFIGURED_ABORT + else: + assert plan.directory_outage + assert await _alive(service, tokens) == set(names) + assert await _audited(store, "auth.ad_session_revoked") == [] + assert await _audited(store, "auth.ad_reconcile_aborted") == [] + refused = await _audited(store, "auth.ad_reconcile_misconfigured") + assert len(refused) == (4 if paged else 0) + assert all(_LEAK not in str(dict(r)["detail"]) for r in refused) + pages = [ + (e[1], e[2]["reason"], e[2]["probed"]) + for e in sink.events + if e[0] == "ad_reconcile_aborted" + ] + expected = ("directory-reconciler-misconfigured", MISCONFIGURED_ABORT, len(names)) + assert pages == ([expected] * 4 if paged else []) + detail = service.directory_reconcile_misconfigured + assert (detail is not None) is paged + assert detail is None or _LEAK not in detail + + async def test_a_busy_account_beside_a_departed_one_is_left_alone_while_the_other_is_revoked( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1016,7 +1076,8 @@ class _ScriptedDc: with ``extended_code``; each search is answered by ``answer(raw request)``. ``searches`` holds every search request's bytes, and ``ended`` is set when a client's socket closes. A reply that is already a whole LDAPMessage is sent as written, under its own messageID, and :data:`_HANG_UP` closes the socket instead of - answering. + answering. ``opened`` counts the sessions accepted, and ``unbound`` the sessions a client + ended with an unbind request. A client reads any close as the server ending the session, so a test that expects exactly that could pass with nothing it scripted read. Two records stop that. ``closed_by_server`` is set @@ -1039,6 +1100,9 @@ def __init__( self.failures: list[Exception] = [] self.closed_by_server = False self.ended = threading.Event() + self.opened = 0 + self.unbound = 0 + self._count_lock = threading.Lock() self._listener = socket.create_server(("127.0.0.1", 0)) self.port: int = self._listener.getsockname()[1] threading.Thread(target=self._accept, daemon=True).start() @@ -1052,6 +1116,8 @@ def _accept(self) -> None: threading.Thread(target=self._session, args=(sock,), daemon=True).start() def _session(self, sock: socket.socket) -> None: + with self._count_lock: + self.opened += 1 try: with sock: try: @@ -1087,6 +1153,9 @@ def _serve(self, sock: socket.socket) -> None: # (_service_session, BACKLOG #3017), may end after the one that hung up. if op != _UNBIND_REQUEST: self.closed_by_server = True + else: + with self._count_lock: + self.unbound += 1 return for reply, controls in replies: if reply == _HANG_UP[0]: diff --git a/tests/test_ldap_timeouts.py b/tests/test_ldap_timeouts.py index 8f460c356..eb2b9ab14 100644 --- a/tests/test_ldap_timeouts.py +++ b/tests/test_ldap_timeouts.py @@ -683,3 +683,4 @@ def test_a_rejected_password_still_unbinds_the_user_connection( "the user connection was NOT unbound on the rejected-password path — the release procedure " "documented in docs/CONNECTIONS.md Table B does not run where it matters most (ASVS 13.1.3)" ) + assert len(rec.unbinds) == 2, f"expected the service and user connections; got {rec.unbinds}" diff --git a/tests/test_lock_notice_off_request_path.py b/tests/test_lock_notice_off_request_path.py index a752657a8..7be5f7800 100644 --- a/tests/test_lock_notice_off_request_path.py +++ b/tests/test_lock_notice_off_request_path.py @@ -16,25 +16,29 @@ import asyncio import json import logging +import threading import time from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from pathlib import Path from typing import Any, Literal +from uuid import uuid4 import pytest from messagefoundry.api import create_managed_app from messagefoundry.auth import service as service_module -from messagefoundry.auth.notifications import ACCOUNT_LOCKED, SecurityEvent +from messagefoundry.auth.notifications import ACCOUNT_LOCKED, PASSWORD_CHANGED, SecurityEvent from messagefoundry.auth.service import AuthService from messagefoundry.config.settings import AlertsSettings, AuthSettings, EgressSettings from messagefoundry.keyed_lock import hold_keyed_lock +from messagefoundry.pipeline.security_notify import SecurityEventNotifier from messagefoundry.store import open_store, sqlite_settings from messagefoundry.store.store import LockoutIncrement, MessageStore, UserRecord from tests._admin_account import ADMIN_USERNAME, create_admin _LOCK_NOTICE = "auth.lock_notice" +_LOCK_NOTICE_UNDELIVERED = "auth.lock_notice_undelivered" #: Longer than the deferred writes' room in the sign-in pad. A write this slow on the request path #: moves the answer to a later slot. _SLOW = service_module._FAILURE_BUDGET_SECONDS * service_module._WRITE_ROOM_SHARE * 2.4 @@ -50,8 +54,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def locks(self) -> list[SecurityEvent]: return [e for e in self.events if e.event_type == ACCOUNT_LOCKED] @@ -77,8 +82,9 @@ async def _harness( async def _seed_other_accounts_notices(store: MessageStore, count: int) -> None: """``count`` recent ``auth.lock_notice`` rows for OTHER accounts, written straight to the table. - The throttle walks every recent row of the action to find this account's, since the audit log - has no actor index, so these are the rows that make its read slow. Inserted in one statement + Before the actor index (vault BACKLOG #3120) the throttle walked every recent row of the + action to find this account's, so these were the rows that made its read slow; the test still + holds the read at a gate, so it does not depend on the read's speed. Inserted in one statement rather than through ``record_audit``, which chains each row and would take minutes. The chain is not under test here.""" db = store._db @@ -100,6 +106,18 @@ async def _seed_other_accounts_notices(store: MessageStore, count: int) -> None: await db.commit() +async def _close(store: MessageStore, service: AuthService) -> None: + """Finish the service's background tasks, then close the store (vault BACKLOG #3120). + + Called from each test's ``finally``, so a test that fails before its own drain leaves no + notice running on, into a closed store and a later test on the shared loop. Bounded, because a + failed test can leave a read held at a gate; a task still running then is cancelled.""" + try: + await service.drain_background(timeout=_STALL_SECONDS) + finally: + await store.close() + + def _in_background(service: AuthService) -> bool: """Whether the running task is one of the service's own background tasks.""" return asyncio.current_task() in service._background_tasks @@ -172,9 +190,10 @@ async def test_the_lock_setting_refusal_stays_in_its_slot_with_a_large_audit_log writes its rows in a room of an eighth of a second, and a throttle read slower than that moved the answer to the next slot and fired the writes-overrun warning. - The audit log holds many ``auth.lock_notice`` rows for other accounts, which is what makes the - real read slow. The read is also held at a gate until the refusal has answered. So it cannot - finish inside the answer, and its walk of those rows cannot compete with the refusal's own + The audit log holds many ``auth.lock_notice`` rows for other accounts, which made the real read + slow before the actor index (vault BACKLOG #3120). The read is also held at a gate until the + refusal has answered. So it cannot finish inside the answer, and its read cannot compete with + the refusal's own writes. The test also checks which task ran the read, so a read on the request path fails it directly. The notice still goes out once the gate opens, and the seeded rows of other accounts do not hold this account's mail back.""" @@ -207,7 +226,7 @@ async def test_the_lock_setting_refusal_stays_in_its_slot_with_a_large_audit_log rows = await _lock_notice_rows(store, user.username) assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": True}] finally: - await store.close() + await _close(store, service) async def test_the_overrun_instrument_fires_for_a_slow_write_still_on_the_path( @@ -238,7 +257,7 @@ async def slow_lock_row(action: str, **kwargs: Any) -> Any: assert _write_overruns(caplog), "the overrun warning did not fire for an on-path write" assert refused_by > service_module._FAILURE_BUDGET_SECONDS + 0.05, refused_by finally: - await store.close() + await _close(store, service) # --- the race --------------------------------------------------------------------------------- @@ -417,7 +436,7 @@ async def test_two_same_kind_locks_at_once_send_one_mail_and_write_one_row( assert len(locked) == 2, "each lock still writes its own auth.account_locked row" assert service._lock_notice_locks == {}, "the queue table kept an entry nobody holds" finally: - await store.close() + await _close(store, service) async def test_the_race_is_real_without_the_queue(monkeypatch: pytest.MonkeyPatch) -> None: @@ -436,7 +455,7 @@ async def test_the_race_is_real_without_the_queue(monkeypatch: pytest.MonkeyPatc assert len(notifier.locks()) == 2 assert len(await _lock_notice_rows(store, user.username)) == 2 finally: - await store.close() + await _close(store, service) async def _peak_reads_across_accounts( @@ -507,15 +526,14 @@ async def counted(*args: Any, **kwargs: Any) -> Any: progress.stalls, ) finally: - await store.close() + await _close(store, service) async def test_one_throttle_read_runs_at_a_time_across_accounts( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Each throttle read walks every recent ``auth.lock_notice`` row, so locks on many accounts at - once would fill the store's read pool that sign-ins wait on. The service runs one at a time. - Every account still gets its mail.""" + """Locks on many accounts at once would each take a read from the store's pool that sign-ins + wait on, so the service runs one throttle read at a time. Every account still gets its mail.""" peak, mailed, locked, stalls = await _peak_reads_across_accounts(monkeypatch, slots=None) assert stalls == [], stalls assert peak == 1, f"{peak} throttle reads ran at once" @@ -580,7 +598,7 @@ async def counted(*args: Any, **kwargs: Any) -> Any: assert peak == 2, "the two kinds' notices queued behind each other" assert sorted(e.detail["lock"] for e in notifier.locks()) == ["second_step", "sign_in"] finally: - await store.close() + await _close(store, service) # --- failure and shutdown --------------------------------------------------------------------- @@ -629,7 +647,7 @@ async def test_a_notice_cancelled_at_shutdown_logs_a_line_that_names_nothing( locked = await store.list_audit(actor=user.username, action="auth.account_locked") assert len(locked) == 1 finally: - await store.close() + await _close(store, service) async def test_a_notice_that_finishes_inside_the_bound_is_not_cancelled( @@ -651,7 +669,7 @@ async def test_a_notice_that_finishes_inside_the_bound_is_not_cancelled( assert len(notifier.locks()) == 1 assert len(await _lock_notice_rows(store, user.username)) == 1 finally: - await store.close() + await _close(store, service) async def test_a_failed_row_write_still_mails_and_logs_the_class_alone( @@ -680,7 +698,7 @@ async def failing(action: str, **kwargs: Any) -> Any: assert _naming(caplog, user) == [], _naming(caplog, user) assert len(notifier.locks()) == 1, "a failed row write dropped the mail" finally: - await store.close() + await _close(store, service) async def test_a_failed_throttle_read_logs_the_class_alone_and_mails( @@ -708,13 +726,13 @@ async def failing(*args: Any, **kwargs: Any) -> Any: assert _naming(caplog, user) == [], _naming(caplog, user) assert len(notifier.locks()) == 1, "a failed read must fail open and mail" finally: - await store.close() + await _close(store, service) class _RefusingNotifier(_FakeNotifier): """A notifier that refuses every hand-off, as a relay whose queue raised would.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise OSError("relay refused the hand-off") @@ -731,15 +749,243 @@ async def test_a_refused_hand_off_does_not_hold_the_next_notice_back() -> None: assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": False}] attempts: list[SecurityEvent] = [] - async def took(event: SecurityEvent) -> None: + async def took(event: SecurityEvent) -> bool: attempts.append(event) + return True notifier.notify = took # type: ignore[method-assign] await service._record_lock(user, "sign_in", _lock(2), client=None) await service.drain_background() assert len(attempts) == 1, "the refused notice held the next one back" finally: - await store.close() + await _close(store, service) + + +def _relay(store: MessageStore | None = None) -> SecurityEventNotifier: + """The shipped notifier, never started, so nothing is sent, with room for ONE queued event.""" + relay = SecurityEventNotifier( + host="smtp.example.test", port=25, sender="mf@example.test", audit=store + ) + relay._queue = asyncio.Queue(maxsize=1) + return relay + + +async def test_the_shipped_notifier_says_whether_it_queued_the_notice() -> None: + """Vault BACKLOG #3120, follow-up 2: ``notify`` answers ``True`` only for an event it queued. + A full queue and a missing address each drop the event, and each answers ``False``.""" + relay = _relay() + queued = SecurityEvent(ACCOUNT_LOCKED, username="lock-a", email="a@example.test") + assert await relay.notify(queued) is True + full = SecurityEvent(ACCOUNT_LOCKED, username="lock-b", email="b@example.test") + assert await relay.notify(full) is False + no_address = SecurityEvent(ACCOUNT_LOCKED, username="lock-c", email=None) + assert await relay.notify(no_address) is False + assert relay._queue.qsize() == 1, "only the first event may have been queued" + + +async def test_a_notice_dropped_on_a_full_queue_does_not_hold_the_next_one_back() -> None: + """Vault BACKLOG #3120, follow-up 2. The shipped notifier's full queue drops a lock notice + without a line, and ``notify`` used to return nothing, so the row read ``mailed: true`` and + the throttle held the next lock's notice back for a day behind a mail that never left. The row + now says ``mailed: false``, and the next lock of the kind is queued once there is room. + + The control is the second lock's own row: with room on the queue it reads ``mailed: true``, so + the instrument can see a notice that was taken.""" + store, service, _fake, user = await _harness() + relay = _relay(store) + relay._queue.put_nowait( + SecurityEvent(ACCOUNT_LOCKED, username="filler", email="f@example.test") + ) + service.attach_security_notifier(relay) + try: + await service._record_lock(user, "sign_in", _lock(1), client=None) + await service.drain_background() + rows = await _lock_notice_rows(store, user.username) + assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": False}] + # BACKLOG #2383: the relay records the drop on its own hidden row too. + await asyncio.gather(*relay._recording) + dropped = await store.list_audit(actor=user.username, action=_LOCK_NOTICE_UNDELIVERED) + assert [json.loads(r["detail"])["reason"] for r in dropped] == ["queue_full"] + + relay._queue.get_nowait() # room again + await service._record_lock(user, "sign_in", _lock(2), client=None) + await service.drain_background() + rows = await _lock_notice_rows(store, user.username) + assert [json.loads(r["detail"])["mailed"] for r in rows] == [True, False], ( + "the dropped notice held the next one back" + ) + assert relay._queue.qsize() == 1, "the second lock's notice was not queued" + finally: + await _close(store, service) + + +async def test_the_shipped_notifier_queues_nothing_once_it_has_closed() -> None: + """Vault BACKLOG #3120: after ``aclose`` nothing drains the queue, so an event ``notify`` put + there would never be sent. It answers ``False`` instead. The control is the same call on the + same notifier before it closed, which answers ``True``.""" + relay = _relay() + relay.start() + try: + assert await relay.notify(SecurityEvent(ACCOUNT_LOCKED, username="a", email="a@x.test")) + relay._queue.get_nowait() # never sent: the drain loop has not run yet + finally: + await relay.aclose() + late = SecurityEvent(ACCOUNT_LOCKED, username="b", email="b@x.test") + assert await relay.notify(late) is False + assert relay._queue.qsize() == 0, "the late event was queued where nothing reads it" + + +async def test_a_closed_notifier_refuses_to_start_again() -> None: + """Vault BACKLOG #3120, review round two. A close can leave its stop sentinel on the queue, so a + drain loop started again could stop on it at once. ``start`` after ``aclose`` therefore raises, + and the notifier goes on refusing events. The control is ``start`` on a notifier never closed, + which raises nothing.""" + relay = _relay() + relay.start() # the control + assert relay._task is not None, "the instrument cannot see a started drain loop" + await relay.aclose() + with pytest.raises(RuntimeError, match="closed"): + relay.start() + assert relay._task is None, "a drain loop was started on a closed notifier" + late = SecurityEvent(ACCOUNT_LOCKED, username="b", email="b@x.test") + assert await relay.notify(late) is False + + +async def test_after_a_cut_close_a_notice_reported_queued_is_sent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Vault BACKLOG #3120, review round two. The teardown bounds the notifier's close, and a close + cut while a send is under way leaves its stop sentinel on the queue. A drain loop started again + then read that sentinel and stopped, and ``notify`` answered ``True`` for an event nothing ever + sent. Whatever ``start`` does after a cut close, a notice reported queued must be sent. + + The control is the event taken before the close: its send was under way when the close was + cut, and once released it goes out and shows in ``sent``.""" + started, release = threading.Event(), threading.Event() + sent: list[list[str]] = [] + + def held_send(**kw: Any) -> None: + if kw["recipients"] == ["a@x.test"]: + started.set() + release.wait(_STALL_SECONDS) + sent.append(kw["recipients"]) + + monkeypatch.setattr("messagefoundry.pipeline.security_notify.send_plain_email", held_send) + relay = _relay() + # Room behind the stop sentinel, so a full queue cannot be what refuses the late event. + relay._queue = asyncio.Queue(maxsize=8) + relay.start() + answered = False + try: + assert await relay.notify(SecurityEvent(PASSWORD_CHANGED, username="a", email="a@x.test")) + assert await asyncio.to_thread(started.wait, _STALL_SECONDS), "the send never started" + with pytest.raises(TimeoutError): # the teardown's bound, cutting the close + await asyncio.wait_for(relay.aclose(), 0.05) + assert relay._queue.qsize() == 1, "the cut close left no stop sentinel to test against" + with suppress(RuntimeError): # refusing is one way to keep the property below + relay.start() + late = SecurityEvent(PASSWORD_CHANGED, username="b", email="b@x.test") + answered = await relay.notify(late) + if answered: + await relay.aclose() # a live drain loop sends what it took before it stops + assert ["b@x.test"] in sent, "a notice reported queued after a cut close was never sent" + finally: + release.set() + await relay.aclose() + deadline = time.monotonic() + _STALL_SECONDS + while ["a@x.test"] not in sent and time.monotonic() < deadline: + await asyncio.sleep(0.01) + assert ["a@x.test"] in sent, "the control's send, under way at the cut, never finished" + if not answered: + assert ["b@x.test"] not in sent, "the late notice was refused, yet a mail went for it" + + +async def test_the_shipped_notifier_queues_nothing_while_it_closes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Vault BACKLOG #3120: once ``aclose`` has begun, an event queued behind its stop sentinel is + never drained, so ``notify`` answers ``False`` from the start of the close, not its end. The + close is held on a send in flight, and the control is that held event, which is sent.""" + started, release = threading.Event(), threading.Event() + sent: list[list[str]] = [] + + def held_send(**kw: Any) -> None: + started.set() + release.wait(_STALL_SECONDS) + sent.append(kw["recipients"]) + + monkeypatch.setattr("messagefoundry.pipeline.security_notify.send_plain_email", held_send) + relay = _relay() + # Room behind the stop sentinel, so a full queue cannot be what refuses the late event. + relay._queue = asyncio.Queue(maxsize=8) + relay.start() + closing: asyncio.Task[None] | None = None + try: + assert await relay.notify(SecurityEvent(PASSWORD_CHANGED, username="a", email="a@x.test")) + assert await asyncio.to_thread(started.wait, _STALL_SECONDS), "the send never started" + closing = asyncio.create_task(relay.aclose()) + await asyncio.sleep(0) # aclose has begun and is waiting on the drain + late = SecurityEvent(PASSWORD_CHANGED, username="b", email="b@x.test") + assert await relay.notify(late) is False + finally: + release.set() + await (closing if closing is not None else relay.aclose()) + assert sent == [["a@x.test"]], "the event taken before the close was not sent" + + +async def test_a_lock_notice_raised_after_the_relay_closed_records_mailed_false( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Vault BACKLOG #3120. At shutdown the teardown can cut the lock-notice drain before it cancels + a pending notice, which then runs on and reaches the security notifier after that has closed. + The notifier used to take it and answer ``True``, so the row read ``mailed: true`` for a mail + that is never sent, and the throttle held the account's next notice of the kind back for a day. + The row now reads ``mailed: false``, so that next lock mails again, and the relay records the + drop with ``reason: closing``. + + The relay is started and its drain loop runs, as at a real shutdown. The control is a second + account's lock before the close, whose notice is sent and whose row reads ``mailed: true``.""" + sent: list[list[str]] = [] + monkeypatch.setattr( + "messagefoundry.pipeline.security_notify.send_plain_email", + lambda **kw: sent.append(kw["recipients"]), + ) + store, service, _fake, user = await _harness() + relay = SecurityEventNotifier( + host="smtp.example.test", port=25, sender="mf@example.test", audit=store + ) + service.attach_security_notifier(relay) + relay.start() + try: + other_id = uuid4().hex + await store.create_user( + user_id=other_id, + username="other-holder", + auth_provider="local", + password_generated=False, + ) + await store.set_user_notify_email(other_id, email="other@example.org") + other_user = await store.get_user(other_id) + assert other_user is not None + await service._record_lock(other_user, "sign_in", _lock(1), client=None) + await service.drain_background() + before = await _lock_notice_rows(store, other_user.username) + assert [json.loads(r["detail"])["mailed"] for r in before] == [True] + + await relay.aclose() + assert sent == [["other@example.org"]], "the control's notice was not sent" + await service._record_lock(user, "sign_in", _lock(1), client=None) + await service.drain_background() + rows = await _lock_notice_rows(store, user.username) + assert [json.loads(r["detail"]) for r in rows] == [{"lock": "sign_in", "mailed": False}] + assert relay._queue.qsize() == 0, "the late notice was queued where nothing reads it" + await asyncio.gather(*relay._recording) + dropped = await store.list_audit(actor=user.username, action=_LOCK_NOTICE_UNDELIVERED) + assert [json.loads(r["detail"])["reason"] for r in dropped] == ["closing"] + assert sent == [["other@example.org"]], "a mail went out after the close" + finally: + await relay.aclose() + await _close(store, service) async def test_after_a_closing_drain_a_notice_runs_inline(monkeypatch: pytest.MonkeyPatch) -> None: @@ -754,17 +1000,17 @@ async def test_after_a_closing_drain_a_notice_runs_inline(monkeypatch: pytest.Mo assert len(notifier.locks()) == 1, "the notice did not run inline after the drain" assert len(await _lock_notice_rows(store, user.username)) == 1 finally: - await store.close() + await _close(store, service) async def test_with_no_notifier_the_row_is_written_inline(monkeypatch: pytest.MonkeyPatch) -> None: """With no notifier wired the throttle reads nothing and writes one ``mailed: false`` row, so that arm stays on the request path and starts no task.""" store = await MessageStore.open(":memory:") + service = AuthService( + store, AuthSettings(lockout_threshold=1, require_mfa=False, notify_security_events=True) + ) try: - service = AuthService( - store, AuthSettings(lockout_threshold=1, require_mfa=False, notify_security_events=True) - ) admin = await create_admin(service) user = await store.get_user(admin.user_id) assert user is not None @@ -773,7 +1019,7 @@ async def test_with_no_notifier_the_row_is_written_inline(monkeypatch: pytest.Mo rows = await _lock_notice_rows(store, user.username) assert [json.loads(r["detail"])["reason"] for r in rows] == ["no_notifier"] finally: - await store.close() + await _close(store, service) # --- the lifespan drains before the store closes ---------------------------------------------- @@ -832,6 +1078,7 @@ async def test_the_lifespan_drains_a_pending_notice_before_the_store_closes( app = _managed_app(db) gate = asyncio.Event() pending_at_drain: list[bool] = [] + auth: AuthService | None = None try: async with app.router.lifespan_context(app): auth = await _lock_in_lifespan(app, monkeypatch, gate=gate) @@ -839,6 +1086,7 @@ async def test_the_lifespan_drains_a_pending_notice_before_the_store_closes( async def open_gate_then_drain() -> None: # The notice is held until the shutdown drain starts, so it is pending there. + assert auth is not None pending_at_drain.append(bool(auth._background_tasks)) gate.set() await real_close() @@ -846,8 +1094,9 @@ async def open_gate_then_drain() -> None: monkeypatch.setattr(auth, "close_background", open_gate_then_drain) finally: gate.set() # never leave a held read behind in the shared loop - # Join anything the lifespan left running, so it cannot outlive the test. - await auth.drain_background() + # Join anything the lifespan left running, so it cannot outlive the test. + if auth is not None: + await auth.drain_background(timeout=_STALL_SECONDS) assert pending_at_drain, "the lifespan never called close_background" assert pending_at_drain == [True], "the notice was not pending when the drain started" rows = await _rows_after_shutdown(db, _LOCK_NOTICE) @@ -864,17 +1113,21 @@ async def test_the_lifespan_control_loses_the_row_without_the_drain( db = tmp_path / "nodrain.db" app = _managed_app(db) gate = asyncio.Event() - async with app.router.lifespan_context(app): - auth = await _lock_in_lifespan(app, monkeypatch, gate=gate) + auth: AuthService | None = None + try: + async with app.router.lifespan_context(app): + auth = await _lock_in_lifespan(app, monkeypatch, gate=gate) - async def skipped() -> None: - return None + async def skipped() -> None: + return None - monkeypatch.setattr(auth, "close_background", skipped) - # The store is closed now. Release the held read and finish the orphaned task here, so it - # meets the closed store deterministically and does not outlive the test. - gate.set() - await auth.drain_background() + monkeypatch.setattr(auth, "close_background", skipped) + finally: + # The store is closed now. Release the held read and finish the orphaned task here, so it + # meets the closed store deterministically and does not outlive the test. + gate.set() + if auth is not None: + await auth.drain_background(timeout=_STALL_SECONDS) assert await _rows_after_shutdown(db, _LOCK_NOTICE) == [], ( "the row survived with the drain neutralised, so the other test does not measure it" ) diff --git a/tests/test_login_new_ip.py b/tests/test_login_new_ip.py index bd610c045..034316369 100644 --- a/tests/test_login_new_ip.py +++ b/tests/test_login_new_ip.py @@ -51,14 +51,15 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True class _RaisingNotifier: """A notifier that always fails, to prove a broken channel cannot refuse a login.""" - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise OSError("synthetic relay failure") @@ -799,8 +800,8 @@ async def test_the_record_is_keyed_on_the_account_not_the_username() -> None: async def test_the_signal_never_reads_the_audit_log(monkeypatch: pytest.MonkeyPatch) -> None: - """Gap 5. The baseline read is one primary-key lookup on the record; ``audit_log``, which has - no actor index, is not read at all. A ``list_audit`` that raises cannot change the verdict.""" + """Gap 5. The baseline read is one primary-key lookup on the record; ``audit_log`` is not read + at all. A ``list_audit`` that raises cannot change the verdict.""" store = await MessageStore.open(":memory:") try: notifier = _FakeNotifier() diff --git a/tests/test_mfa.py b/tests/test_mfa.py index e8e63f175..01a7f90f0 100644 --- a/tests/test_mfa.py +++ b/tests/test_mfa.py @@ -59,8 +59,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _store() -> MessageStore: @@ -474,19 +475,19 @@ async def test_parallel_wrong_credentials_cannot_evade_the_account_lockout( also pins the one-notice-per-lockout contract. """ store = await _store() + threshold, burst = 3, 5 + notifier = _FakeNotifier() + service = AuthService( + store, + AuthSettings( + mfa_verify_min_elapsed_seconds=0, + lockout_threshold=threshold, + lockout_minutes=15, + mfa_recovery_code_count=3, + ), + security_notifier=notifier, + ) try: - threshold, burst = 3, 5 - notifier = _FakeNotifier() - service = AuthService( - store, - AuthSettings( - mfa_verify_min_elapsed_seconds=0, - lockout_threshold=threshold, - lockout_minutes=15, - mfa_recovery_code_count=3, - ), - security_notifier=notifier, - ) identity, token, password = await login_admin(service) counts = _count_verifies(monkeypatch, service) @@ -548,10 +549,11 @@ async def test_parallel_wrong_credentials_cannot_evade_the_account_lockout( # The per-account queue leaves no entry behind once every attempt has left it. assert service._credential_locks == {}, "the per-account lock entry must not leak" - # Arms 2 and 3 set locks too; finish their background notices before the store closes - # (BACKLOG #2216), so none runs on into a later test on the shared loop. - await service.drain_background() finally: + # Arms 2 and 3 set locks too; finish their background notices before the store closes + # (BACKLOG #2216), so none runs on into a later test on the shared loop. In the finally, + # so a failed assertion above cannot skip it (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() diff --git a/tests/test_mfa_directory_recheck.py b/tests/test_mfa_directory_recheck.py index a5f5eef8d..09116ba28 100644 --- a/tests/test_mfa_directory_recheck.py +++ b/tests/test_mfa_directory_recheck.py @@ -33,6 +33,7 @@ AdPrincipal, DirectoryAnswer, LdapReferralError, + LdapUnansweredError, ) from messagefoundry.auth.service import DIRECTORY_UNCONFIRMED, AuthService, DirectoryRefusal from messagefoundry.auth.tokens import hash_token @@ -182,8 +183,14 @@ async def test_a_directory_the_account_is_not_confirmed_in_refuses_the_renewal( (ValueError("synthetic: malformed objectGUID"), "probe_fault"), # A network-class error that escaped the directory layer is still unreachable. (TimeoutError("synthetic: read timed out"), "unavailable"), + # BACKLOG #3010: a code that points at the configuration, and one that does not. + ( + LdapUnansweredError("synthetic: noSuchObject", code=32, code_name="noSuchObject"), + "misconfigured", + ), + (LdapUnansweredError("synthetic: busy", code=51, code_name="busy"), "unavailable"), ], - ids=["referral", "key-error", "value-error", "timeout"], + ids=["referral", "key-error", "value-error", "timeout", "no-such-object", "busy"], ) async def test_a_lookup_that_raises_refuses_the_renewal_rather_than_failing( store: MessageStore, diff --git a/tests/test_notify_email_confinement.py b/tests/test_notify_email_confinement.py index 8db1e3cd4..e8a24b5bd 100644 --- a/tests/test_notify_email_confinement.py +++ b/tests/test_notify_email_confinement.py @@ -52,8 +52,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def _no_mfa(**overrides: Any) -> AuthSettings: diff --git a/tests/test_postgres_store.py b/tests/test_postgres_store.py index fa53c5045..b9fae7068 100644 --- a/tests/test_postgres_store.py +++ b/tests/test_postgres_store.py @@ -90,6 +90,8 @@ "reference", "reference_version", "sessions", + "session_lineage", # vault BACKLOG #3028 (no FK): a link left behind would lead a later test's + # supersession into a session it never retired "webauthn_credentials", # ADR 0068: FK to users(id) — must clear before users "known_login_addresses", # vault BACKLOG #2145: FK to users(id), cleared before users "user_roles", @@ -5472,13 +5474,21 @@ async def fetch(sql: str, params: tuple[object, ...]) -> list[dict[str, object]] async def test_session_supersession_contract(store) -> None: - """BACKLOG #2146: supersede_session revokes and returns in one statement, so a rotation after it - fails closed. Extra-free shared contract, run here as on SQLite.""" + """BACKLOG #2146: supersede_session revokes and returns in one transaction, so a rotation after + it fails closed. Extra-free shared contract, run here as on SQLite.""" from tests._session_rotation_contract import assert_session_supersession_contract await assert_session_supersession_contract(store) +async def test_session_lineage_contract(store) -> None: + """Vault BACKLOG #3028: a rotation links the hash it retired, the supersession follows the + links inside its window, and the purge drops them past it. Extra-free shared contract.""" + from tests._session_rotation_contract import assert_session_lineage_contract + + await assert_session_lineage_contract(store) + + async def test_session_cap_contract(store) -> None: """BACKLOG #1900: the per-user cap counts only LIVE sessions. What this leg executes that no other does: the liveness clauses respelled for ``$n``, with ``$1`` reused in the UPDATE and in diff --git a/tests/test_provision_first_administrator.py b/tests/test_provision_first_administrator.py index a3eacb6ca..8486e273f 100644 --- a/tests/test_provision_first_administrator.py +++ b/tests/test_provision_first_administrator.py @@ -1418,8 +1418,9 @@ def __init__(self) -> None: # How many notices were queued when the drain ran: a drain before the notice loses it. self.queued_at_close: int | None = None - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def start(self) -> None: self.started = True @@ -1804,7 +1805,7 @@ async def test_a_notifier_that_raises_is_recorded_as_no_hand_off() -> None: """The row is written after the notice, so a notifier failure cannot be recorded as dispatched.""" class _Raising(_Recorder): - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: raise RuntimeError("synthetic notifier failure") store = await MessageStore.open(":memory:") diff --git a/tests/test_reauth_lockout.py b/tests/test_reauth_lockout.py index fecdf0b33..3e8442341 100644 --- a/tests/test_reauth_lockout.py +++ b/tests/test_reauth_lockout.py @@ -71,8 +71,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True def _actions(feed: list[dict[str, Any]], action: str) -> list[dict[str, Any]]: @@ -540,13 +541,13 @@ async def test_a_wrong_current_password_counts_and_is_audited_once() -> None: async def test_the_reauth_crossing_attempt_audits_account_locked_once_and_notifies() -> None: store = await MessageStore.open(":memory:") + notifier = _FakeNotifier() + service = AuthService( + store, + AuthSettings(lockout_threshold=3, require_mfa=False), + security_notifier=notifier, + ) try: - notifier = _FakeNotifier() - service = AuthService( - store, - AuthSettings(lockout_threshold=3, require_mfa=False), - security_notifier=notifier, - ) await _local_user(store) identity, token = await _signed_in(service) for _ in range(3): @@ -570,6 +571,8 @@ async def test_the_reauth_crossing_attempt_audits_account_locked_once_and_notifi assert notices[0].detail == {"failed_attempts": 3, "lock": "sign_in", "cycle": 1} assert notices[0].client_ip == "10.0.0.9" finally: + # Again here, so a failure above cannot leave the notice running (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() diff --git a/tests/test_session_sql_placeholders.py b/tests/test_session_sql_placeholders.py index 0743b01fa..e20c85b7f 100644 --- a/tests/test_session_sql_placeholders.py +++ b/tests/test_session_sql_placeholders.py @@ -57,6 +57,10 @@ "rotate_session": lambda s: s.rotate_session(_HASH, new_token_hash=_OTHER), "revoke_session": lambda s: s.revoke_session(_HASH, now=_NOW), "supersede_session": lambda s: s.supersede_session(_HASH, now=_NOW), + # Vault BACKLOG #3028: nothing is revoked, so the walk reads the lineage once and stops. + "supersede_session_lineage": lambda s: s.supersede_session( + _HASH, now=_NOW, lineage_since=_NOW - 360 + ), "revoke_user_sessions": lambda s: s.revoke_user_sessions("u", now=_NOW), "revoke_user_sessions_except": lambda s: s.revoke_user_sessions( "u", except_token_hash=_HASH, now=_NOW @@ -69,6 +73,9 @@ ), "purge_expired_sessions": lambda s: s.purge_expired_sessions(now=_NOW), "purge_expired_sessions_idle": lambda s: s.purge_expired_sessions(now=_NOW, idle_seconds=1800), + "purge_expired_sessions_lineage": lambda s: s.purge_expired_sessions( + now=_NOW, idle_seconds=1800, lineage_seconds=360 + ), } @@ -223,7 +230,14 @@ async def test_sqlserver_rotate_reads_its_output_rowset_not_the_row_count( store = _nocount_store(seen, output_rows) assert await store.rotate_session(_HASH, new_token_hash=_OTHER) is bool(output_rows) # The read comes first (BACKLOG #2592); the rewrite is the statement that reports. - assert "OUTPUT inserted.token_hash" in seen[-1][0] + assert "OUTPUT inserted.token_hash" in seen[1][0] + # The lineage link follows the rewrite only when the rowset says it landed (vault BACKLOG + # #3028), so a refused rotation links nothing. + links = [sql for sql, _ in seen if sql.startswith("INSERT INTO session_lineage")] + assert len(links) == (1 if output_rows else 0) + # The generic placeholder test never lands a rotation, so the link's binding is checked here. + for sql, params in seen: + assert sql.count("?") == len(params), f"{len(params)} parameters for:\n{sql}" @pytest.mark.parametrize("revoked", [0, 3]) @@ -246,9 +260,10 @@ async def bump(cur: Any, user_id: str) -> None: async def test_sqlserver_supersede_returns_the_output_row_by_column_name() -> None: - """``supersede_session`` reads its ``OUTPUT inserted.*`` row through ``_execute_output`` - (BACKLOG #2283), which pairs each value with its column name. Read by position instead, a - reordered column list would fill the record from the wrong columns.""" + """``supersede_session`` reads its ``OUTPUT inserted.*`` row through ``_fetch_dicts``, inside + its own transaction (BACKLOG #2283, vault BACKLOG #3028), which pairs each value with its column + name. Read by position instead, a reordered column list would fill the record from the wrong + columns.""" columns = ( "user_id", "token_hash", @@ -334,6 +349,10 @@ async def fetchrow(self, sql: str, *args: Any) -> Any: return {"session_epoch": 0} return None + async def fetchval(self, sql: str, *args: Any) -> Any: + self._seen.append((sql, args)) + return None + def transaction(self) -> Any: @asynccontextmanager async def _txn() -> AsyncIterator[None]: diff --git a/tests/test_session_supersession.py b/tests/test_session_supersession.py index 6b8b01b43..7eb01fe9c 100644 --- a/tests/test_session_supersession.py +++ b/tests/test_session_supersession.py @@ -208,7 +208,8 @@ async def test_a_rotation_racing_the_supersession_cannot_keep_the_session_live( # FIRST store call that names the presented hash, whichever call that is, so it measures the # gap rather than one method name. Once that call has run, the session must end: the rotation # fails closed, or its new token does not validate. A rotation that commits BEFORE that call is - # the stated limit, pinned in tests/_session_rotation_contract.py. If a later change reads the + # followed through the session lineage (vault BACKLOG #3028), which + # tests/test_session_supersession_lineage.py pins. If a later change reads the # presented hash earlier in the sign-in, this fires there, and that read must not decide the # revoke, or the gap is back. service = await _service(store) @@ -244,12 +245,14 @@ def _stepping() -> float: reads[0] += 1 return base + reads[0] * 0.001 - async def _touched_first(token_hash: str, *, now: float) -> object: + async def _touched_first( + token_hash: str, *, now: float, lineage_since: float | None = None + ) -> object: touched = time.time() assert touched > now, "the stepping clock did not separate the touch from the revoke stamp" await store.touch_session(token_hash, now=touched) touches.append(token_hash) - return await real_supersede(token_hash, now=now) + return await real_supersede(token_hash, now=now, lineage_since=lineage_since) monkeypatch.setattr(store, "supersede_session", _touched_first) monkeypatch.setattr(time, "time", _stepping) @@ -281,10 +284,10 @@ async def test_a_session_live_when_the_revoke_began_is_audited_though_it_lapsed_ # password hash cannot eat the margin on a slow runner. clock = {"now": record.last_used_at + 30 * 60 - 2.0} # 2 seconds of idle time left - async def _slow(token_hash: str, *, now: float) -> object: + async def _slow(token_hash: str, *, now: float, lineage_since: float | None = None) -> object: if token_hash == prior_hash: clock["now"] += 4.0 # the call returns 2 seconds past the idle limit - return await real_supersede(token_hash, now=now) + return await real_supersede(token_hash, now=now, lineage_since=lineage_since) monkeypatch.setattr(time, "time", lambda: clock["now"]) monkeypatch.setattr(store, "supersede_session", _slow) diff --git a/tests/test_session_supersession_lineage.py b/tests/test_session_supersession_lineage.py new file mode 100644 index 000000000..e372997c3 --- /dev/null +++ b/tests/test_session_supersession_lineage.py @@ -0,0 +1,406 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""A sign-in ends the session its presented token BECAME, for a bounded time (vault BACKLOG #3028). + +Engine PR 2028 (BACKLOG #2146) made the supersession's read and revoke one store operation, so a +rotation that starts after it fails closed. It left one case open: an elevation that FINISHED before +it. The presented hash then named no session, the supersession ended nothing, and the session lived +on under its new token. A rotation now records a link from the retired hash to its successor, and +the supersession follows that link to the live end, inside its own store transaction. + +These pin the service half. The store half, on every backend, is +``tests/_session_rotation_contract.py``'s ``assert_session_lineage_contract``. +""" + +from __future__ import annotations + +import ast +import asyncio +import re +import sqlite3 +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import closing +from pathlib import Path +from urllib.parse import parse_qsl, urlsplit +from uuid import uuid4 + +import pytest + +from messagefoundry.auth import Role +from messagefoundry.auth.oidc import FederatedPrincipal +from messagefoundry.auth.service import AuthService +from messagefoundry.auth.tokens import hash_token +from messagefoundry.store.session_lineage import LINEAGE_MAX_HOPS, revoke_through_lineage +from messagefoundry.store.store import MessageStore, SessionRecord +from tests._ast_sites import code_names, code_strings, parse_source +from tests.test_session_supersession import ( + PW, + _live, + _oidc_settings, + _service, + _superseded, + _token, +) + + +@pytest.fixture +async def store(tmp_path: Path) -> AsyncIterator[MessageStore]: + s = await MessageStore.open(tmp_path / "lineage.db") + yield s + await s.close() + + +async def _elevate(service: AuthService, token: str) -> str: + """Complete a real password step-up on ``token`` and return the token it was rotated to.""" + identity = await service.identity_for_token(token, activity=False) + assert identity is not None + elevation = await service.reauth(identity, PW, token=token) + assert elevation.ok and elevation.token is not None, elevation + assert elevation.token != token + return elevation.token + + +async def test_an_elevation_that_finished_before_the_sign_in_does_not_keep_the_session_live( + store: MessageStore, +) -> None: + # Closing item 3 of the row. The elevation runs to completion, so its rotation has committed + # before the sign-in's supersession statement runs. The browser still presents the OLD token, + # which now names no session row. + service = await _service(store) + prior = await _token(service) + rotated = await _elevate(service, prior) + assert await store.get_session(hash_token(prior)) is None, "the rotation did not retire it" + assert await _live(service, rotated) + + new = await _token(service, supersedes=prior) + + assert not await _live(service, rotated), ( + "the sign-in replaced this browser's session, but the session lives on under its new token" + ) + assert await _live(service, new) + assert await _superseded(store) == [ + { + "scope": "superseded", + "session": hash_token(rotated)[:12], + "presented": hash_token(prior)[:12], + "actor": "op", + } + ] + + +async def test_an_elevation_during_the_identity_provider_round_trip_is_followed( + store: MessageStore, monkeypatch: pytest.MonkeyPatch +) -> None: + # The federated leg, the widest window: the START leg stages the presented hash, an elevation + # in another tab rotates the session while the browser is at the identity provider, and the + # cookie-less callback supersedes the staged hash. The IdP exchange is replaced at the one seam + # the service calls, as tests/test_session_supersession.py does. + service = await _service(store, **_oidc_settings()) + await service.set_ad_group_map([("cn=mf-ops,dc=x", Role.OPERATOR.value)], actor="admin") + jdoe_id = uuid4().hex + await store.create_user( + user_id=jdoe_id, + username="jdoe", + auth_provider="ad", + directory_object_id="guid-jdoe", + password_generated=False, + ) + await service.bind_federated_subject( + jdoe_id, "S-1-5-21-fed", expected_issuer=None, expected_subject=None, actor="admin" + ) + prior = await _token(service) + + def _exchange(*_a: object, **_k: object) -> FederatedPrincipal: + return FederatedPrincipal( + username="jdoe", + subject="S-1-5-21-fed", + issuer="https://idp.example", + amr=("pwd", "mfa"), + acr=None, + expires_at=time.time() + 600, + auth_time=time.time(), + ) + + monkeypatch.setattr(service, "_exchange_and_validate", _exchange) + flow_id, url = await service.begin_oidc_login( + client=None, public_origin="https://ops.example", prior_session=prior + ) + rotated = await _elevate(service, prior) # finishes while the browser is at the IdP + outcome = await service.complete_oidc_login( + flow_id=flow_id, + state=dict(parse_qsl(urlsplit(url).query))["state"], + code="authcode", + client=None, + public_origin="https://ops.example", + ) + assert outcome.ok, outcome + assert not await _live(service, rotated), "the federated callback did not follow the lineage" + + +async def test_the_link_is_followed_over_several_elevations(store: MessageStore) -> None: + service = await _service(store) + prior = await _token(service) + end = prior + for _ in range(3): + end = await _elevate(service, end) + await _token(service, supersedes=prior) + assert not await _live(service, end), "the chain was not followed to its live end" + + +async def test_a_link_older_than_the_window_is_not_followed( + store: MessageStore, monkeypatch: pytest.MonkeyPatch +) -> None: + service = await _service(store, session_idle_timeout_minutes=240) + window = service.supersession_lineage_seconds + assert window >= service.oidc_flow_ttl_seconds, "the window must outlast the longest sign-in" + prior = await _token(service) + rotated = await _elevate(service, prior) + real = time.time + monkeypatch.setattr(time, "time", lambda: real() + window + 5) + assert await _live(service, rotated) + await _token(service, supersedes=prior) + assert await _live(service, rotated), "a link past its window ended a session" + assert await _superseded(store) == [] + + +def _links(path: Path) -> list[tuple[str, str]]: + with closing(sqlite3.connect(path)) as db: + return [ + (str(r[0]), str(r[1])) + for r in db.execute("SELECT retired_hash, successor_hash FROM session_lineage") + ] + + +async def test_a_retired_token_authenticates_nothing_while_its_link_is_live( + store: MessageStore, tmp_path: Path +) -> None: + # Closing item 2. The link exists and is inside its window, and the retired token still + # resolves to nothing, on every path that reads a session by a presented token. + service = await _service(store) + prior = await _token(service) + rotated = await _elevate(service, prior) + assert _links(tmp_path / "lineage.db") == [(hash_token(prior), hash_token(rotated))] + assert await service.identity_for_token(prior) is None + assert await service.identity_for_token(prior, activity=False) is None + assert await store.get_session(hash_token(prior)) is None + assert await _live(service, rotated), "presenting the retired token ended its successor" + + +async def test_a_token_whose_hash_links_to_a_live_session_authenticates_nothing( + store: MessageStore, tmp_path: Path +) -> None: + # The same, from the other side: a link row a database writer forged from a hash whose token it + # holds to someone else's live session. The validator reads `sessions` only, so the forged + # token is refused and the session it points at is untouched. + service = await _service(store) + victim = await _token(service, "other") + forged = "a-token-nobody-was-ever-issued" + with closing(sqlite3.connect(tmp_path / "lineage.db")) as db: + db.execute( + "INSERT INTO session_lineage (retired_hash, successor_hash, retired_at) VALUES (?,?,?)", + (hash_token(forged), hash_token(victim), time.time()), + ) + db.commit() + assert await service.identity_for_token(forged) is None + assert await _live(service, victim) + + +_BACKENDS = ("store.py", "postgres.py", "sqlserver.py") +_STORE_DIR = Path(__file__).resolve().parents[1] / "messagefoundry" / "store" +_LINEAGE = re.compile(r"\bsession_lineage\b|SESSION_LINEAGE_|\brevoke_through_lineage\b") + + +def _functions_naming_the_lineage(source: str) -> set[str]: + """Every top-level function or method whose CODE names the lineage table, one of its + statements, or the shared walk; a docstring or comment that mentions them does not count. A + nested function counts as the method it sits in.""" + tree = parse_source(source) + named: set[str] = set() + tops: list[ast.stmt] = list(tree.body) + for cls in tree.body: + if isinstance(cls, ast.ClassDef): + tops.extend(cls.body) + for node in tops: + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and any( + _LINEAGE.search(text) for text in (*code_names(node), *code_strings(node)) + ): + named.add(node.name) + return named + + +@pytest.mark.parametrize("backend", _BACKENDS) +def test_only_the_rotation_the_supersession_and_the_purge_touch_the_lineage(backend: str) -> None: + # Closing item 2, structurally, and only this much: in the three backend modules, no method but + # these three spells the table, one of its statements or the shared walk in code. A read through + # a statement under another name, or from a new module, is outside what this proves; the + # behavioural tests above cover the validator itself. + source = (_STORE_DIR / backend).read_text(encoding="utf-8") + assert _functions_naming_the_lineage(source) == { + "rotate_session", + "supersede_session", + "purge_expired_sessions", + } + + +def test_the_service_follows_the_lineage_from_the_supersession_alone() -> None: + # `_load_session` and the validator never pass a lineage window; only the sign-in's + # supersession does. `_end_lapsed_session` shares the store call and passes none. + source = (Path(__file__).resolve().parents[1] / "messagefoundry/auth/service.py").read_text( + encoding="utf-8" + ) + tree = ast.parse(source) + passing: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.AsyncFunctionDef | ast.FunctionDef): + for call in ast.walk(node): + if isinstance(call, ast.Call) and any( + k.arg == "lineage_since" for k in call.keywords + ): + passing.add(node.name) + assert passing == {"_supersede_session_hash"} + assert not _LINEAGE.search(source) + + +async def test_the_lineage_reaches_no_further_than_one_supersession(store: MessageStore) -> None: + # A followed link ends the live end once. Presenting the retired token again finds a revoked row + # and writes nothing. + service = await _service(store) + prior = await _token(service) + await _elevate(service, prior) + await _token(service, supersedes=prior) + await _token(service, supersedes=prior) + assert len(await _superseded(store)) == 1 + + +async def test_the_lapsed_credential_revoke_does_not_follow_the_lineage( + store: MessageStore, +) -> None: + # `_end_lapsed_session` shares `supersede_session`. It revokes the row it was handed, and a row a + # rotation re-keyed first is not reported as ended; the next presentation of the new token is + # refused by the same gate. So it passes no lineage window. + service = await _service(store) + prior = await _token(service) + rotated = await _elevate(service, prior) + await service._end_lapsed_session(hash_token(prior), "op", at="session", client=None) + assert await _live(service, rotated) + + +async def test_the_reaper_purges_links_by_the_window_it_is_given( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The lifespan hands the reaper the service's window, and the reaper must pass it to the purge, + # or no link is ever deleted. + from messagefoundry.api import app as api_app + + monkeypatch.setattr(api_app, "_SESSION_REAP_INTERVAL", 0) + monkeypatch.setattr(api_app, "_SESSION_REAP_FIRST_DELAY", 0) + seen: list[float | None] = [] + done = asyncio.Event() + + class _Store: + async def purge_expired_sessions( + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, + ) -> None: + seen.append(lineage_seconds) + done.set() + + task = asyncio.create_task( + api_app._session_reaper(_Store(), idle_seconds=60.0, lineage_seconds=420.0) # type: ignore[arg-type] + ) + try: + await asyncio.wait_for(done.wait(), timeout=5) + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert seen[0] == 420.0 + + +def test_the_lifespan_hands_the_reaper_the_supersession_window() -> None: + source = (Path(__file__).resolve().parents[1] / "messagefoundry/api/app.py").read_text( + encoding="utf-8" + ) + reapers = [c for c in ast.walk(parse_source(source)) if isinstance(c, ast.Call)] + passing = [ + c + for c in reapers + if isinstance(c.func, ast.Name) + and c.func.id == "_session_reaper" + and any( + k.arg == "lineage_seconds" + and isinstance(k.value, ast.Attribute) + and k.value.attr == "supersession_lineage_seconds" + for k in c.keywords + ) + ] + assert len(passing) == 1, "the lifespan no longer passes the window to the reaper" + + +# --- the shared walk, with no database: a rotation that lands mid-walk, and the hop bound ---------- + + +def _record(token_hash: str) -> SessionRecord: + return SessionRecord( + token_hash=token_hash, + user_id="u", + created_at=0.0, + expires_at=1.0, + last_used_at=0.0, + revoked_at=1.0, + client=None, + ) + + +async def test_a_rotation_landing_mid_walk_costs_a_step_not_the_revoke() -> None: + # On the server backends each step is its own statement, so a rotation can commit between a + # step's lineage read and the next step's revoke. Here `b` is re-keyed to `c` exactly then. The + # walk's revoke of `b` matches nothing, it reads `b`'s new link, and it ends `c`. + live = {"b"} + links = {"a": "b"} + calls: list[str] = [] + + async def _revoke(h: str) -> SessionRecord | None: + calls.append(f"revoke {h}") + if h in live: + live.discard(h) + return _record(h) + return None + + async def _successor(h: str, since: float) -> str | None: + calls.append(f"follow {h}") + found = links.get(h) + if h == "a": # the rotation of b -> c commits right after this read + live.discard("b") + live.add("c") + links["b"] = "c" + return found + + ended = await revoke_through_lineage( + "a", lineage_since=0.0, revoke=_revoke, successor=_successor + ) + assert ended is not None and ended.token_hash == "c" + assert calls == ["revoke a", "follow a", "revoke b", "follow b", "revoke c"] + + +async def test_the_walk_stops_at_its_hop_bound() -> None: + async def _successor(h: str, since: float) -> str | None: + return str(int(h) + 1) + + def _revoke_only(target: int) -> Callable[[str], Awaitable[SessionRecord | None]]: + async def _revoke(h: str) -> SessionRecord | None: + return _record(h) if h == str(target) else None + + return _revoke + + reach = await revoke_through_lineage( + "0", lineage_since=0.0, revoke=_revoke_only(LINEAGE_MAX_HOPS), successor=_successor + ) + assert reach is not None and reach.token_hash == str(LINEAGE_MAX_HOPS) + past = await revoke_through_lineage( + "0", lineage_since=0.0, revoke=_revoke_only(LINEAGE_MAX_HOPS + 1), successor=_successor + ) + assert past is None, "the walk followed more links than its bound" diff --git a/tests/test_sqlserver_store.py b/tests/test_sqlserver_store.py index be5388168..9fff273b9 100644 --- a/tests/test_sqlserver_store.py +++ b/tests/test_sqlserver_store.py @@ -114,6 +114,8 @@ async def store() -> AsyncIterator[object]: # silently miscounts an unrelated one "messages", "sessions", + "session_lineage", # vault BACKLOG #3028 (no FK): a stale link would lead a later + # test's supersession into a session it never retired "webauthn_credentials", # ADR 0068: FK to users(id) — must clear before users "known_login_addresses", # vault BACKLOG #2145: per-account, cleared before users "user_roles", @@ -5856,13 +5858,21 @@ async def fetch(sql: str, params: tuple[Any, ...]) -> list[dict[str, Any]]: async def test_session_supersession_contract(store) -> None: - """BACKLOG #2146: supersede_session revokes and returns in one statement, so a rotation after it - fails closed. Extra-free shared contract, run here as on SQLite.""" + """BACKLOG #2146: supersede_session revokes and returns in one transaction, so a rotation after + it fails closed. Extra-free shared contract, run here as on SQLite.""" from tests._session_rotation_contract import assert_session_supersession_contract await assert_session_supersession_contract(store) +async def test_session_lineage_contract(store) -> None: + """Vault BACKLOG #3028: a rotation links the hash it retired, the supersession follows the + links inside its window, and the purge drops them past it. Extra-free shared contract.""" + from tests._session_rotation_contract import assert_session_lineage_contract + + await assert_session_lineage_contract(store) + + async def test_session_cap_contract(store) -> None: """BACKLOG #1900: the per-user cap counts only LIVE sessions. What this leg executes that no other does: ``TOP (?)`` ahead of the subquery's WHERE, which binds ``keep`` before the liveness diff --git a/tests/test_store_pool_acquire_timeout.py b/tests/test_store_pool_acquire_timeout.py index 43cc501ed..7c4c61d42 100644 --- a/tests/test_store_pool_acquire_timeout.py +++ b/tests/test_store_pool_acquire_timeout.py @@ -379,7 +379,11 @@ async def test_postgres_post_commit_session_writes_wait_out_a_busy_pool( assert not task.done(), f"{name} gave up on a busy pool after its caller's write committed" pool.gate.set() assert await asyncio.wait_for(task, timeout=5) == expected - assert pool.statements and pool.statements[-1].startswith("UPDATE sessions") + assert any(s.startswith("UPDATE sessions") for s in pool.statements), pool.statements + if name == "rotate_session": + # The rotation's lineage link (vault BACKLOG #3028) follows its re-key on the same borrow: + # the stand-in pool hands out one connection, and it recorded both statements. + assert pool.statements[-1].startswith("INSERT INTO session_lineage"), pool.statements class _MssqlSessionCursor: @@ -474,7 +478,13 @@ async def bump(cur: Any, user_id: str) -> None: assert not task.done(), f"{name} gave up on a busy pool after its caller's write committed" pool.gate.set() assert await asyncio.wait_for(task, timeout=5) == expected - assert pool.statements and pool.statements[-1].startswith("UPDATE sessions") + assert any(s.startswith("UPDATE sessions") for s in pool.statements), pool.statements + if name == "rotate_session": + # The rotation's lineage link (vault BACKLOG #3028) follows its re-key in the same + # transaction, on the same post-commit borrow, as in the Postgres twin above. + assert pool.statements[-1].startswith("INSERT INTO session_lineage"), pool.statements + else: + assert pool.statements[-1].startswith("UPDATE sessions"), pool.statements assert pool.released == [pool.conn] and pool.checked_out == [] assert store._acquire_wait.summary().count == 1, "the post-commit wait left the histogram" diff --git a/tests/test_suspicious_login_security_events.py b/tests/test_suspicious_login_security_events.py index 9b092d134..2770a7577 100644 --- a/tests/test_suspicious_login_security_events.py +++ b/tests/test_suspicious_login_security_events.py @@ -54,8 +54,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _store() -> MessageStore: @@ -211,11 +212,11 @@ async def test_a_lockout_crossed_on_the_second_factor_is_in_the_users_feed() -> async def test_the_lockout_row_is_written_when_a_notifier_is_wired_too() -> None: store = await _store() + notifier = _FakeNotifier() + service = AuthService( + store, AuthSettings(lockout_threshold=3, require_mfa=False), security_notifier=notifier + ) try: - notifier = _FakeNotifier() - service = AuthService( - store, AuthSettings(lockout_threshold=3, require_mfa=False), security_notifier=notifier - ) await _local_user(store) for _ in range(3): await service.login("bob", WRONG_PASSWORD) @@ -228,4 +229,6 @@ async def test_the_lockout_row_is_written_when_a_notifier_is_wired_too() -> None # The count stays in the notice, beside the lock kind and its cycle count (ADR 0197). assert notices[0].detail == {"failed_attempts": 3, "lock": "sign_in", "cycle": 1} finally: + # Again here, so a failure above cannot leave the notice running (vault BACKLOG #3120). + await service.drain_background(timeout=20.0) await store.close() diff --git a/tests/test_task_resilience.py b/tests/test_task_resilience.py index 67d3f9c08..158ae9584 100644 --- a/tests/test_task_resilience.py +++ b/tests/test_task_resilience.py @@ -171,7 +171,11 @@ async def test_session_reaper_survives_purge_error(monkeypatch: pytest.MonkeyPat class _Store: async def purge_expired_sessions( - self, *, now: float | None = None, idle_seconds: float | None = None + self, + *, + now: float | None = None, + idle_seconds: float | None = None, + lineage_seconds: float | None = None, ) -> None: calls["n"] += 1 if calls["n"] == 1: diff --git a/tests/test_webauthn.py b/tests/test_webauthn.py index f7efbc320..2316303ec 100644 --- a/tests/test_webauthn.py +++ b/tests/test_webauthn.py @@ -44,8 +44,9 @@ class _FakeNotifier: def __init__(self) -> None: self.events: list[SecurityEvent] = [] - async def notify(self, event: SecurityEvent) -> None: + async def notify(self, event: SecurityEvent) -> bool: self.events.append(event) + return True async def _service( diff --git a/tests/test_webauthn_store.py b/tests/test_webauthn_store.py index 799466f99..7ade299a4 100644 --- a/tests/test_webauthn_store.py +++ b/tests/test_webauthn_store.py @@ -112,3 +112,15 @@ async def test_session_supersession_contract_sqlite(tmp_path: Path) -> None: await assert_session_supersession_contract(store) finally: await store.close() + + +async def test_session_lineage_contract_sqlite(tmp_path: Path) -> None: + """SQLite leg of the vault BACKLOG #3028 session lineage contract, shared with the live Postgres + / SQL Server suites.""" + store = await MessageStore.open(tmp_path / "lin.db") + try: + from tests._session_rotation_contract import assert_session_lineage_contract + + await assert_session_lineage_contract(store) + finally: + await store.close()