From 6f2f4f5081d6f6fa503f5b0c0c83b2d6900ccdef Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:32:19 -0500 Subject: [PATCH 01/27] The start-up credential probe runs off the event loop (BACKLOG #3164) The api/app.py lifespan called AuthService.probe_credential_generation synchronously, so the password generator's policy screens ran on the loop thread at start. A pathological password_extra_context_words list makes the generator retry many times. The lifespan now awaits asyncio.to_thread(auth.probe_credential_generation), the same shape as the Kerberos preflight a few lines below, and keeps its except guard. The method stays synchronous, so the existing synchronous test in tests/test_generated_credential_failure.py is untouched, and the verify CLI caller, which has no loop, is untouched too. tests/test_credential_probe_off_loop.py drives the real lifespan and records where the generator's screen ran; it fails on the old code. A second test pins that the guard still logs a raise and the start continues. --- messagefoundry/api/app.py | 6 +- tests/test_credential_probe_off_loop.py | 95 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 tests/test_credential_probe_off_loop.py diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 052fb355f5..3fe2b485f8 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -10489,9 +10489,11 @@ async def _audit_upload_prune(meta: UploadedFileMeta) -> None: # ADR 0197 Amendment A, Manager decision 2026-09-29: learn at start, not at the first # account creation, that the password policy leaves no credential to issue. It logs # an ERROR and returns, and is guarded like the census: a policy-level fact must - # never refuse the start, whatever a later policy clause raises. + # never refuse the start, whatever a later policy clause raises. It runs off the loop + # (vault BACKLOG #3164): a pathological context-word list makes the generator retry + # its policy screens many times, and the loop already serves the API by now. try: - auth.probe_credential_generation() + await asyncio.to_thread(auth.probe_credential_generation) except Exception: # noqa: BLE001 -- see the comment above _log.exception( "the credential-generation probe could not run; startup continues" diff --git a/tests/test_credential_probe_off_loop.py b/tests/test_credential_probe_off_loop.py new file mode 100644 index 0000000000..fc008e4a93 --- /dev/null +++ b/tests/test_credential_probe_off_loop.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""The start-up credential-generation probe runs off the event loop (vault BACKLOG #3164). + +``AuthService.probe_credential_generation`` runs the temporary-credential generator once at start. +A pathological ``[auth].password_extra_context_words`` list makes that generator retry its policy +screens many times, so the lifespan must not run it on the loop thread, the way the request-path +callers already do not (BACKLOG #2359). These drive the real lifespan and record where the +generator's screen ran. + +Severity is conditional (CLAUDE.md section 0): zero deployments, so a held loop is what a deploying +site would meet at start, never a live exposure. +""" + +from __future__ import annotations + +import asyncio +import logging +from pathlib import Path + +import pytest +from fastapi import FastAPI + +from messagefoundry.api import create_managed_app +from messagefoundry.auth import service as service_module +from messagefoundry.auth.policy import PasswordPolicy +from messagefoundry.config.settings import ( + AlertsSettings, + AuthSettings, + EgressSettings, + SecurityEnforcement, + SecuritySettings, + StoreSettings, +) +from messagefoundry.store.crypto import generate_key + + +def _app(db: Path) -> FastAPI: + """An engine that starts on an empty store: ``warn`` lets the ADR 0167 gate pass with no + Administrator, so the lifespan reaches the probe.""" + return create_managed_app( + store_settings=StoreSettings(path=str(db), encryption_key=generate_key()), + poll_interval=0.05, + auth_settings=AuthSettings(), + alerts_settings=AlertsSettings( + email_smtp_host="smtp.example.invalid", email_from="alerts@example.invalid" + ), + security_settings=SecuritySettings(enforcement=SecurityEnforcement.WARN), + egress_settings=EgressSettings(deny_by_default=False), + ) + + +async def _start(app: FastAPI) -> None: + async with app.router.lifespan_context(app): + pass + + +def test_the_lifespan_runs_the_generator_off_the_loop_thread( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + on_loop: list[bool] = [] + + def recording(policy: PasswordPolicy) -> str | None: + # A worker thread has no running loop; the loop thread does. + try: + asyncio.get_running_loop() + except RuntimeError: + on_loop.append(False) + else: + on_loop.append(True) + return None + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service_module, "credential_generation_problem", recording) + asyncio.run(_start(_app(tmp_path / "probe.db"))) + # The count is the control: an empty list would pass the assertion below vacuously. + assert on_loop == [False], "the probe ran once, and not on the event loop thread" + + +def test_a_probe_that_raises_off_the_loop_still_lets_the_start_continue( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """The guard still wraps the call now that it is awaited: a raise in the worker thread comes + back through ``asyncio.to_thread`` and is logged, and the start does not refuse.""" + + def broken(policy: PasswordPolicy) -> str | None: + raise ValueError("a later policy clause raised") + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service_module, "credential_generation_problem", broken) + with caplog.at_level(logging.ERROR, logger="messagefoundry.api.app"): + asyncio.run(_start(_app(tmp_path / "probe.db"))) # must not raise + assert any( + "the credential-generation probe could not run" in r.getMessage() for r in caplog.records + ) From 251080a9f1bac5bb80b671fa5bbf5e7a13d78b3a Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:32:52 -0500 Subject: [PATCH 02/27] Directory role and disable notices name the directory, not an administrator (BACKLOG #3121) The directory reconciler's ACCOUNT_DISABLED and ROLES_CHANGED notices said an administrator made the change and closed with "If this was you". _apply_reconcile_revocation now tags its notice detail with source "directory", and so does the sign-in role re-sync in _complete_ad_login, so the two directory ROLES_CHANGED notices read alike. In pipeline/security_notify.py the source tag alone now decides from_directory, for any kind, so a new directory notice cannot fall back to the administrator wording by missing from a list. ROLES_CHANGED and ACCOUNT_DISABLED take directory descriptions that name no administrator, gain the directory line, and close with "If you did not expect this change". The administrator path keeps its wording, as the row says; that wording is not judged correct (BACKLOG #1141, vault #3072 item 10). Tests pin the descriptions whole, not only words the subject would satisfy: the directory variants, the unchanged administrator variants, the sign-in address line on the re-sync notice, and the existing directory_sessions_ended description. The reconciler and login-path tests assert the source tag. docs/SECURITY.md says how directory notices are worded. --- docs/SECURITY.md | 11 ++- messagefoundry/auth/notifications.py | 8 ++- messagefoundry/auth/service.py | 16 +++-- messagefoundry/pipeline/security_notify.py | 29 ++++++-- tests/test_ad_session_reconcile.py | 8 ++- tests/test_auth_service.py | 4 ++ tests/test_security_notify.py | 84 +++++++++++++++++++++- 7 files changed, 141 insertions(+), 19 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index d0a9314cc6..089cc02aa2 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4384,9 +4384,14 @@ Users are notified of security-relevant changes to their account through **two** alert distribution list). Fired on: account **lockout** and the **first successful login after ≥3 failed attempts**, or a step-up re-auth that clears such a run (suspicious-login signals, 6.3.5); and **password change**, **email change**, **role change**, **account disable**, and a **directory rename** of the username (credential changes, - 6.3.7). When the directory session reconciler ends an account's sessions because it could not - match the account or could not read its `userAccountControl`, it sends `directory_sessions_ended`, - not `account_disabled` (vault BACKLOG #2140). + 6.3.7). + A notice for a change the directory made names the directory and no administrator, and it closes + with "If you did not expect this change" rather than "If this was you". That covers at least the + directory rename, the directory's email change, a role change applied by the session reconciler + or by a directory sign-in's re-sync, and the reconciler's `account_disabled` when it read the + account disabled (vault BACKLOG #3121). When the reconciler ends an account's sessions because it + could not match the account or could not read its `userAccountControl`, it sends + `directory_sessions_ended`, not `account_disabled` (vault BACKLOG #2140). The rename notice, `username_changed`, names the old and the new name, and is sent only when the new name was written (BACKLOG #2017). An **unreplaced temporary password** near its deadline (ASVS 6.4.5, BACKLOG #2007) sends two reminders, beside the operator's `initial_credential_expiring` diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index a772bc1381..9d108998b4 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -50,7 +50,9 @@ TEMPORARY_CREDENTIAL_EXPIRING = "temporary_credential_expiring" TEMPORARY_CREDENTIAL_EXPIRING_FOR_ISSUER = "temporary_credential_expiring_issuer" EMAIL_CHANGED = "email_changed" # 6.3.7 — the account's email address was changed -ROLES_CHANGED = "roles_changed" # 6.3.7 — an admin changed the account's roles +# 6.3.7 -- the account's roles changed: an administrator's act, or the directory's, which the +# reconciler and the sign-in re-sync tag with ``detail["source"] == "directory"`` (vault BACKLOG #3121). +ROLES_CHANGED = "roles_changed" # 6.3.7 -- the directory renamed the account and the engine copied the new name onto the row # (BACKLOG #2017). When it is sent is stated once, at the send site in # ``AuthService._refresh_cached_username``. ``detail`` carries ``old_username``, ``new_username`` and @@ -67,7 +69,9 @@ # address the account held BEFORE the takeover, because a moved address belongs to the operator who # ran the command, not to the holder being told. FIRST_ADMINISTRATOR_TAKEOVER = "first_administrator_takeover" -ACCOUNT_DISABLED = "account_disabled" # 6.3.7 — an admin disabled the account +# 6.3.7 -- an administrator disabled the account, or the reconciler read it disabled in the +# directory, tagged ``detail["source"] == "directory"`` (vault BACKLOG #3121). +ACCOUNT_DISABLED = "account_disabled" # 6.3.7 -- the directory session reconciler ended the account's sessions for a reason that is NOT a # read disabled bit: the probe could not match the account (``directory_absent``: not found, or its # stored id did not match or could not be parsed), or its ``userAccountControl`` could not be read diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index da69512cfb..c05f6b0c9e 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -5448,12 +5448,14 @@ async def _complete_ad_login( # A directory-pushed privilege change is the same privilege change to the same user as a # local one, so notify the affected user out-of-band too (ASVS 6.3.7), matching set_roles(). # Best-effort; the change is also visible in the audited /me/security-events feed. + # ``source`` words it for the directory, as the reconciler's notice is (vault BACKLOG + # #3121): no administrator made this change, so "if this was you" cannot apply. await self._notify_security( ROLES_CHANGED, username=user.username, email=user.notify_email, client=client, - detail={"roles": role_ids}, + detail={"roles": role_ids, "source": "directory"}, ) ad_roles = _roles_from_ids(role_ids) ad_custom_permissions = await self._custom_permissions_for_ids(role_ids) @@ -7382,9 +7384,13 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat # ``ad_session_revoked`` alert record it. # # ACCOUNT_DISABLED only when the pass READ the disabled bit (vault BACKLOG #2140). That - # notice says an administrator disabled the account, which an absent account or an unreadable - # attribute does not establish, so every other whole-account reason, including any added - # later, gets the neutral DIRECTORY_SESSIONS_ENDED. + # notice says the directory reports the account disabled, which an absent account or an + # unreadable attribute does not establish, so every other whole-account reason, including + # any added later, gets the neutral DIRECTORY_SESSIONS_ENDED. + # + # ``source`` makes the renderer word ROLES_CHANGED and ACCOUNT_DISABLED for the directory, + # with no administrator in them and no "if this was you" (vault BACKLOG #3121). The + # login-path re-sync tags its ROLES_CHANGED the same way. if revocation.role_ids is not None: kind = ROLES_CHANGED elif revocation.reason == reconcile.REVOKE_REASONS[reconcile.ProbeOutcome.DISABLED]: @@ -7396,7 +7402,7 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat kind, username=user.username, email=user.notify_email, - detail={"reason": revocation.reason}, + detail={"reason": revocation.reason, "source": "directory"}, ) return True diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 1652eae709..9c82556f3d 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -166,6 +166,19 @@ ), } +# Vault BACKLOG #3121. A directory-driven role change or disable names no administrator: the +# directory made it. The directory line and the "If you did not expect this change" closing follow +# from ``from_directory`` in :func:`_build_body`, so these say only what changed. The disable reads +# what the reconciler did, which is end the sessions; it does not claim MessageFoundry disabled +# anything. +_DIRECTORY_DESCRIPTIONS = { + ROLES_CHANGED: "Your account's roles were changed.", + ACCOUNT_DISABLED: ( + "Your organization's directory reports your account as disabled, so MessageFoundry ended " + "your sessions." + ), +} + _LOCAL_REMEDY = ( "ask your MessageFoundry administrator for a password reset, or the host operator to run " @@ -257,9 +270,9 @@ def _build_body(event: SecurityEvent) -> str: event.event_type == NOTIFY_EMAIL_SET and event.detail.get("set_by") == "administrator" ) # BACKLOG #1139, #2291: the directory made this change, not the holder and not the console. - from_directory = event.event_type in (EMAIL_CHANGED, USERNAME_CHANGED) and ( - event.detail.get("source") == "directory" - ) + # The sender's tag decides, whatever the kind (vault BACKLOG #3121), so a new directory notice + # cannot fall back to the administrator wording by missing from a list here. + from_directory = event.detail.get("source") == "directory" if moved_by_admin: description = ( "An administrator changed the address that receives security notices for your account." @@ -268,6 +281,9 @@ def _build_body(event: SecurityEvent) -> str: description = ( "An administrator set this address to receive security notices about your account." ) + elif from_directory and event.event_type in _DIRECTORY_DESCRIPTIONS: + # Only these two kinds name an administrator in their own text; the others need no swap. + description = _DIRECTORY_DESCRIPTIONS[event.event_type] else: description = _DESCRIPTIONS.get( event.event_type, "A security event occurred on your account." @@ -437,7 +453,9 @@ def _build_body(event: SecurityEvent) -> str: # and a bare "Source IP" line would read as the address of whoever made it. Measured # 2026-10-06, the directory notices that pass an address come from ``_upsert_ad_user``, on # a directory sign-in to this account, which copies the change down BEFORE the sign-in is - # finished: a later step can still refuse it, or it can still owe a second factor. So the + # finished: a later step can still refuse it, or it can still owe a second factor. Since + # vault BACKLOG #3121 the role re-sync in ``_complete_ad_login`` is one more such sender, + # and it too runs inside a directory sign-in before the session is issued. So the # line says a sign-in was under way, never that one succeeded. Nothing on the event says # the address came from a sign-in, so a new sender passing another kind of address on a # directory event would make this line false; such a sender must change it. @@ -482,7 +500,8 @@ def _build_body(event: SecurityEvent) -> str: ) ): # A directory rename or email change is an administrator's act, so "if this was you" - # cannot apply to it (the email half is BACKLOG #2291). Nor to a federated link or unlink: + # cannot apply to it (the email half is BACKLOG #2291), nor to a directory role change or + # disable (vault BACKLOG #3121). Nor to a federated link or unlink: # both routes refuse an administrator changing their own account's binding. Nor to the # engine ending sessions on a directory answer (vault BACKLOG #2140). closing = "If you did not expect this change, contact your MessageFoundry administrator." diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index b8d7952136..5dbe0199e8 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -3470,8 +3470,8 @@ async def test_a_scope_revocation_sends_no_account_disabled_notice() -> None: #: vault BACKLOG #2140. The notice each revocation reason sends, decided once. ``None`` is no notice. -#: ACCOUNT_DISABLED says an administrator disabled the account, so only the reason that READ the -#: disabled bit may send it. +#: ACCOUNT_DISABLED says the directory reports the account disabled, so only the reason that READ +#: the disabled bit may send it. _NOTICE_FOR_REASON: dict[str, str | None] = { "directory_disabled": ACCOUNT_DISABLED, "directory_absent": DIRECTORY_SESSIONS_ENDED, @@ -3511,7 +3511,9 @@ async def test_each_revocation_reason_sends_its_decided_notice(reason: str) -> N expected = _NOTICE_FOR_REASON[reason] assert [e.event_type for e in notifier.sent] == ([] if expected is None else [expected]) if expected is not None: - assert notifier.sent[0].detail == {"reason": reason} + # ``source`` words ROLES_CHANGED and ACCOUNT_DISABLED for the directory (vault + # BACKLOG #3121); every reconciler notice carries it. + assert notifier.sent[0].detail == {"reason": reason, "source": "directory"} finally: await store.close() diff --git a/tests/test_auth_service.py b/tests/test_auth_service.py index 0955336679..5d94184cc8 100644 --- a/tests/test_auth_service.py +++ b/tests/test_auth_service.py @@ -828,6 +828,8 @@ def role_changes() -> list[SecurityEvent]: assert len(changes) == 2 assert changes[-1].username == "jdoe" and changes[-1].email == "jdoe@example.org" assert changes[-1].detail.get("roles") == ["viewer"] + # Worded for the directory, as the reconciler's notice is (vault BACKLOG #3121). + assert changes[-1].detail.get("source") == "directory" finally: await store.close() @@ -945,6 +947,8 @@ async def test_clearing_the_profile_address_leaves_the_account_still_notifiable( later = notifier.events[before:] assert [e.event_type for e in later] == [ROLES_CHANGED] assert later[0].email == "bob@example.org" + # An administrator's act keeps the administrator wording (vault BACKLOG #3121). + assert "source" not in later[0].detail finally: await store.close() diff --git a/tests/test_security_notify.py b/tests/test_security_notify.py index be36691ee5..beea972807 100644 --- a/tests/test_security_notify.py +++ b/tests/test_security_notify.py @@ -15,6 +15,7 @@ import pytest from messagefoundry.auth.notifications import ( + ACCOUNT_DISABLED, ACCOUNT_LOCKED, ADMIN_NEW_IP, DIRECTORY_SESSIONS_ENDED, @@ -24,6 +25,7 @@ NOTIFY_EMAIL_SET, PASSWORD_RESET, RECOVERY_CODE_USED, + ROLES_CHANGED, TEMPORARY_CREDENTIAL_EXPIRING, TEMPORARY_CREDENTIAL_EXPIRING_FOR_ISSUER, USERNAME_CHANGED, @@ -210,7 +212,11 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di body = _build_body(SecurityEvent(DIRECTORY_SESSIONS_ENDED, username="jdoe", email=None)) text = (_SUBJECTS[DIRECTORY_SESSIONS_ENDED] + "\n" + body).lower() assert "a security event occurred on your account." not in text - assert "directory" in text and "sessions" in text + # The description itself, not words the subject alone would satisfy (vault BACKLOG #3121). + assert ( + "A check against your organization's directory ended your MessageFoundry sessions." + in body.splitlines() + ) for claim in ("disabled", "deleted", "if this was you"): assert claim not in text assert body.endswith( @@ -218,6 +224,82 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di ) +#: vault BACKLOG #3121. The description each directory-driven kind carries, pinned whole. +_DIRECTORY_TEXT = { + ROLES_CHANGED: "Your account's roles were changed.", + ACCOUNT_DISABLED: ( + "Your organization's directory reports your account as disabled, so MessageFoundry ended " + "your sessions." + ), +} +_DIRECTORY_LINE = ( + "This change came from your organization's directory, not from the MessageFoundry console." +) +_DIRECTORY_CLOSING = "If you did not expect this change, contact your MessageFoundry administrator." +_ADMIN_TEXT = { + ROLES_CHANGED: "Your account's roles were changed by an administrator.", + ACCOUNT_DISABLED: "Your account was disabled by an administrator.", +} +_IF_THIS_WAS_YOU = ( + "If this was you, no action is needed. If not, contact your MessageFoundry administrator." +) + + +@pytest.mark.parametrize("kind", [ROLES_CHANGED, ACCOUNT_DISABLED]) +def test_a_directory_role_or_disable_notice_names_the_directory_and_no_administrator( + kind: str, +) -> None: + """vault BACKLOG #3121. The reconciler and the sign-in re-sync tag these ``source`` directory: + no administrator made the change and the holder did nothing, so neither the administrator + wording nor "if this was you" may appear.""" + body = _build_body( + SecurityEvent( + kind, + username="jdoe", + email="j@example.org", + detail={"reason": "directory_disabled", "source": "directory"}, + ) + ) + lines = body.splitlines() + assert _DIRECTORY_TEXT[kind] in lines + assert _DIRECTORY_LINE in lines + assert _ADMIN_TEXT[kind] not in lines + assert "if this was you" not in body.lower() + assert body.endswith(_DIRECTORY_CLOSING) + + +@pytest.mark.parametrize("kind", [ROLES_CHANGED, ACCOUNT_DISABLED]) +def test_an_administrator_role_or_disable_notice_keeps_its_wording(kind: str) -> None: + """vault BACKLOG #3121, closing item 2: the notice an administrator's own act sends keeps its + text. Kept by the row's instruction, not judged correct: "if this was you" on an administrator's + act is the shape BACKLOG #1141 fixed for the reset notice, and vault #3072 item 10 names.""" + body = _build_body(SecurityEvent(kind, username="jdoe", email="j@example.org")) + lines = body.splitlines() + assert _ADMIN_TEXT[kind] in lines + assert _DIRECTORY_TEXT[kind] not in lines + assert "directory" not in body.lower() + assert body.endswith(_IF_THIS_WAS_YOU) + + +def test_the_sign_in_role_re_sync_notice_says_where_the_sign_in_came_from() -> None: + """vault BACKLOG #3121. The sign-in re-sync passes the client address, and a directory notice + must not print it as the address of whoever made the change.""" + body = _build_body( + SecurityEvent( + ROLES_CHANGED, + username="jdoe", + email="j@example.org", + client_ip="203.0.113.7", + detail={"roles": ["viewer"], "source": "directory"}, + ) + ) + assert "Source IP" not in body + assert ( + "MessageFoundry picked up this change during a sign-in to your account from 203.0.113.7." + in body.splitlines() + ) + + def test_body_says_a_directory_repoint_came_from_the_directory() -> None: """BACKLOG #1139 (ASVS 6.3.7): where the change came from decides what the reader can DO. A directory-driven repoint is not editable in the console, so an unexplained one reads as a From c2337e6134cb6b9ee128614bdeb26624fe1c90d6 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:34:45 -0500 Subject: [PATCH 03/27] TOTP enrolment follow-ups from PR 2080 (BACKLOG #3164 #3121 #3113) This commit is the BACKLOG #3113 layer; the two before it are #3164 and #3121. Item 1: provision_first_administrator's refusal when enable_totp matches no row no longer tells the operator to remove the entry it showed. Under the re-stage race (vault #3025) another run can turn on this run's secret, so the text says to keep the entry until a sign-in shows which works. A race test drives two runs on one row in a fixed order and measures that the loser's secret is the live one. Items 2 and 7: in confirm_mfa_enrollment a store refusal of revoke_session is logged via write_audit_soft and the auth.mfa_enroll_refused row is still written; session_ended records the revoke's outcome instead of a constant. Items 3 and 5: the Elevation docstring, the "a retry works" comment and two long SECURITY.md sentences now say what the code does. Item 4: the docs/ASVS-L2-PHASE0-CHANGES.md TOTP key row no longer says a live code proves the copy that is enabled; it names #3025. Item 6: test_webui asserts enable_totp's result. Items 8 and 9 are declined in code comments: 8 would undo #1902's rotate-first order; 9 needs enable_totp to return a reason on all three store backends. Proposed PR title: Auth follow-ups: start-up probe off the loop, directory notice wording, TOTP enrolment refusal fixes (BACKLOG #3164 #3121 #3113) Proposed banner, #3164: BUILT on engine branch b257-d1. The lifespan awaits asyncio.to_thread(auth.probe_credential_generation) with its guard kept; tests/test_credential_probe_off_loop.py proves the generator runs off the loop thread. Proposed banner, #3121: BUILT on engine branch b257-d1. Reconciler and sign-in re-sync ROLES_CHANGED and ACCOUNT_DISABLED notices carry source directory and render directory wording with the "If you did not expect this change" closing; administrator wording kept; tests pin the descriptions. Proposed banner, #3113: BUILT on engine branch b257-d1. Items 1 to 7 done with tests (race test for item 1, raise test for items 2 and 7); items 8 and 9 declined in writing in confirm_mfa_enrollment. The PHASE0 line is corrected; no vault docs/security file quotes it, so no cell re-read is owed unless the scorecard holder finds one. --- docs/ASVS-L2-PHASE0-CHANGES.md | 2 +- docs/SECURITY.md | 12 ++-- messagefoundry/auth/service.py | 55 +++++++++++---- .../tests/test_webui.py | 3 +- tests/_admin_account.py | 9 +-- tests/test_mfa.py | 53 ++++++++++++++- tests/test_provision_first_administrator.py | 68 +++++++++++++++++++ 7 files changed, 177 insertions(+), 25 deletions(-) diff --git a/docs/ASVS-L2-PHASE0-CHANGES.md b/docs/ASVS-L2-PHASE0-CHANGES.md index 653036ec04..4bcdd77e74 100644 --- a/docs/ASVS-L2-PHASE0-CHANGES.md +++ b/docs/ASVS-L2-PHASE0-CHANGES.md @@ -468,7 +468,7 @@ is right. | Off-box log-forward client key | 4, on the same condition as the API TLS server key | Loads the combined PEM in `_build_tls_context` ([`logging_setup.py`](../messagefoundry/logging_setup.py)). There is no passphrase setting | As for the API TLS server key. The file's permissions are the key's only protection, so setting them is the site's job | | Native API client key | 4, on the same condition as the API TLS server key | Holds no copy. The client process loads it (`_build_verify_context` in [`apiclient/client.py`](../messagefoundry/apiclient/client.py)) | Every function in this row belongs to the operator of the client machine | | Non-production self-signed key | Two keys, each 4, private authentication key, like the API TLS placeholder key. The `cert self-signed` key is a placeholder a site may serve. The load-test harness pair is also test keying material under section 8.1.4 | Mints both with `make_self_signed` in [`pki.py`](../messagefoundry/pki.py), the CLI key on request. The CLI writes its key through `_write_private_key`, which refuses to overwrite an existing file. The harness writes its pair with a plain write and `chmod`, as the lifecycle row above says. Nothing is audited | The `cert self-signed` key is a placeholder key, like the API TLS placeholder key: a site may serve it until it supplies a chain. Set its cryptoperiod under section 5.3.6 item 4 through `--days`, because the engine never renews it. Replace it with an operator-supplied chain, then delete both files. The load-test harness pair is test keying material, which section 8.1.4 says shall not be used operationally, and the engine does not stop that | -| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code proves the user's copy, and audits `auth.mfa_enrolled`. **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | +| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable itself checks only that some secret is still staged and TOTP is still off, not that the staged secret is the one the code proved, so a second enrolment start that re-stages the secret inside the confirm's window can turn on a secret no code proved (vault BACKLOG #3025, open; corrected under vault BACKLOG #3113). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | | Anonymizer re-identification salt | None of the 19 types in 5.1.1 fits. It keys a hash that pseudonymizes one dataset. The nearest fit is 5.1.2's *other secret information*, so this row borrows 5.3.7's rule for that and takes no cryptoperiod from 5.3.6 | Never mints or stores it. `Keyer` in [`anon/keying.py`](../messagefoundry/anon/keying.py) refuses a visibly weak salt, and the export reads it from the environment for one run | Mint it with a CSPRNG, one per dataset, and hold it with one person. Section 5.3.7 says other secret information shall not be kept longer than necessary, so its period is one dataset. Keep no backup unless the dataset must be regenerated, and destroy it when that need ends | | Sealed-cache key | 6, symmetric data-encryption key. It protects the confidentiality and integrity of decoded cache values in the engine's own memory | Mints it per process (`_Sealer` in [`store/sealed_cache.py`](../messagefoundry/store/sealed_cache.py)) and uses it only to seal and open cache values, with each value bound to its cache key as associated data. Pre-activation and active are one step, and the key is destroyed with the process, so its cryptoperiod (5.3.6) is the process lifetime. There is no backup or archive of it (Table 7 is moot) | Protect the engine process's memory: the in-use residual in `docs/PHI.md` §3 and the host requirements in §10 apply, because this key sits in the same memory as the values it seals | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 089cc02aa2..cd9bb91528 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2768,11 +2768,13 @@ way past the sign-in lock (ADR 0197 Amendment A). Such an account's first factor An administrator clears a lost authenticator via `POST /users/{id}/reset-mfa` (which also revokes the user's sessions). A confirm turns TOTP on only while a secret is still staged and TOTP is still off (BACKLOG -#2224). If a reset clears the secret during the confirm, or another confirm turns TOTP on first, -the confirm turns nothing on and returns no recovery codes. It ends the session it just rotated and -answers as a lost session: `401` on `POST /me/mfa/confirm`, and the sign-in page in the web -console. It is audited `auth.mfa_enroll_refused`, whose `reason` is `secret_cleared`, -`already_enabled` or `account_gone`, read after the refusal. The check is that some secret is +#2224). A reset can clear the secret during the confirm, or another confirm can turn TOTP on +first. Either way the confirm turns nothing on and returns no recovery codes. It ends the session +it just rotated and answers as a lost session: `401` on `POST /me/mfa/confirm`, and the sign-in +page in the web console. It is audited `auth.mfa_enroll_refused`. Its `reason` is +`secret_cleared`, `already_enabled` or `account_gone`. That reason is read after the refusal, so a +later write can blur it. Its `session_ended` is false when the store refused to end the rotated +session; the token is still not handed back (vault BACKLOG #3113). The check is that some secret is staged, not that it is the one the confirm verified: a second enrolment start that re-stages the secret inside the same window is not caught by it. With `[security].require_mfa` on — **the default since BACKLOG #187 (secure-by-default, including the diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index c05f6b0c9e..485a2b6cab 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -824,9 +824,12 @@ class Elevation: the session was ended because its temporary password lapsed, with the proof checked or not, as that ``auth.temp_password_expired`` row records; or (``confirm_mfa_enrollment`` only, BACKLOG #2224) the proof was good but turning TOTP on matched no row, so the confirm ended - the session it had just rotated, as its ``auth.mfa_enroll_refused`` row records. Fails - CLOSED: no token is handed back. Held apart from a wrong proof so a route answers 401 rather than re-prompting on a - session that no longer exists. The ``auth.reauth`` row's ``session_revoked`` says which. + the session it had just rotated, as its ``auth.mfa_enroll_refused`` row records. That row's + ``session_ended`` is false when the store refused the revoke (vault BACKLOG #3113): the + rotated session then still exists, but its token was never handed back. Fails CLOSED: no + token is handed back. Held apart from a wrong proof so a route answers 401 rather than + re-prompting on a session that is gone or unreachable. For a password re-auth, the + ``auth.reauth`` row's ``session_revoked`` says which cause applied. ``recovery_codes`` is populated only by :meth:`AuthService.confirm_mfa_enrollment` (shown once). @@ -835,7 +838,7 @@ class Elevation: correct code as incorrect. A password re-proof is never refused for a lock (BACKLOG #1138). ``ok`` is DERIVED, not stored, and that is load-bearing rather than tidiness. Held as a field it - was a second spelling of ``token is not None`` -- true of all 19 constructions -- so the type + was a second spelling of ``token is not None`` -- true of every construction then -- so the type could represent a state the system never produces, mypy could not narrow ``token`` from ``ok``, and every consuming site paid ``if not elevation.ok or elevation.token is None``: a second clause whose only job was to re-derive the first in a form the checker accepts. As a property the @@ -3120,13 +3123,20 @@ async def provision_first_administrator( # Conditional (BACKLOG #2224): a factor clear by another run's repair branch, or another # run that enabled TOTP first, since the secret was staged above makes it match no row. # Refused before the role, so the row stays roleless and a re-run completes it. + # + # The words must not say to remove the entry this run showed (vault BACKLOG #3113). + # Staging is unconditional, so the last run to stage owns the staged secret, and another + # run's enable can turn on THIS run's secret before this enable is refused (the re-stage + # race, vault BACKLOG #3025). So this run cannot tell whether its entry is live. if not await self._store.enable_totp(user_id, recovery_code_hashes=hashes): raise FirstAdministratorRefused( "the authenticator enrolment was changed while this command ran, so this run " - "did not turn TOTP on and granted no role; remove the authenticator entry it " - "showed. " + "did not turn TOTP on and granted no role. Another run may have turned on the " + "secret this run showed, so keep the authenticator entry it showed until a " + "sign-in to this account shows which entry works. " "If another provision-admin run is in progress, let it finish. Then run the " - "command again" + "command again: it reads that account and either completes it or says why it " + "cannot" ) if notify_email is not None: # Unconditional rather than fresh-path-only, because the invariant "the supplied address @@ -10066,7 +10076,8 @@ async def confirm_mfa_enrollment( still OFF (BACKLOG #1902), so a lost session never leaves MFA on with codes nobody saw. So does a good code whose activation matches no row, because a reset cleared the staged secret or a second confirm enabled first (BACKLOG #2224). That confirm ends the session it rotated, - hands back no codes, and audits ``auth.mfa_enroll_refused``. + hands back no codes, and audits ``auth.mfa_enroll_refused``, whose ``session_ended`` says + whether the store took the revoke (vault BACKLOG #3113). This is one of the two legs that turn an MFA-pending session into an MFA-satisfied one for a FIRST enrolment, so it rotates for the same reason ``verify_mfa`` does: without it a pre-MFA @@ -10117,7 +10128,10 @@ async def confirm_mfa_enrollment( # the rotation has succeeded; on a lost session MFA stays off and the staged secret stays put, # and the user signs in again and confirms with a code from a later step (this one is spent). # The cost: if enable_totp itself fails after a good rotation, the new token is never handed - # back, so the user is signed out with MFA still off. That fails closed, and a retry works. + # back, so the user is signed out with MFA still off. That fails closed. A retry is not + # always a plain one (vault BACKLOG #3113): it needs a code from a later step, since this one + # is spent, and where the enable matched no row (#2224, below) a reset may have cleared the + # staged secret, so the user starts the enrolment again. # Stamp against the OLD hash, then rotate — never the other way round. async with self._sealed_stamp(hash_token(token)): await self._store.mark_session_mfa_verified(hash_token(token)) @@ -10140,10 +10154,25 @@ async def confirm_mfa_enrollment( # # Known cost: ``_elevated`` already ran the session cap for the rotated session, so # at the cap that run may have ended the account's oldest full session, and it stays - # ended. The order is #1902's, rotate first and enable last, and is kept. - await self._store.revoke_session(hash_token(elevation.token)) + # ended. The order is #1902's, rotate first and enable last, and is kept. Vault BACKLOG + # #3113 item 8 is declined for that reason: enabling first would undo #1902. + # + # A store refusal of the revoke is logged, not raised, so the refusal row below is + # still written (vault BACKLOG #3113). ``session_ended`` records what the revoke did. + # The token is not handed back either way, so the caller still cannot use the session. + rotated = hash_token(elevation.token) + session_ended = await write_audit_soft( + lambda: self._store.revoke_session(rotated), + log=_log, + message="a refused TOTP enrolment for %s could not end the session it rotated", + args=lambda: (scrub_log_argument(identity.username),), + errors=_audit_write_errors(), + defects=AUDIT_WRITE_DEFECTS, + ) # The reason is the row's state read AFTER the refusal, not the write's own verdict, - # so a later write can blur it; it is a pointer for an operator, not a proof. + # so a later write can blur it; it is a pointer for an operator, not a proof. Vault + # BACKLOG #3113 item 9 is declined: the write's own verdict would need ``enable_totp`` + # to return a reason on all three store backends. after = await self._load_user(identity.user_id) if after is None: reason = "account_gone" @@ -10154,7 +10183,7 @@ async def confirm_mfa_enrollment( await self._audit( "auth.mfa_enroll_refused", actor=identity.username, - detail=_json({"reason": reason, "session_ended": True}), + detail=_json({"reason": reason, "session_ended": session_ended}), client=client, ) return Elevation(session_lost=True) diff --git a/packaging/messagefoundry-webconsole/tests/test_webui.py b/packaging/messagefoundry-webconsole/tests/test_webui.py index eebfef122e..a1a9bb67d8 100644 --- a/packaging/messagefoundry-webconsole/tests/test_webui.py +++ b/packaging/messagefoundry-webconsole/tests/test_webui.py @@ -6254,7 +6254,8 @@ async def test_reauth_both_factors_enrolled_renders_code_and_passkey(engine: Eng secret = totp_mod.generate_secret() await service.store.set_totp_secret(boss_id, secret=secret) - await service.store.enable_totp(boss_id, recovery_code_hashes=[]) + # Conditional since BACKLOG #2224, so the premise is asserted (vault BACKLOG #3113 item 6). + assert await service.store.enable_totp(boss_id, recovery_code_hashes=[]) is True await _cookie_login(c, "boss") # fresh MFA-pending session boss_creds = await service.store.list_webauthn_credentials(boss_id) diff --git a/tests/_admin_account.py b/tests/_admin_account.py index 3fce51f421..df1e6331cc 100644 --- a/tests/_admin_account.py +++ b/tests/_admin_account.py @@ -129,16 +129,17 @@ async def create_local_user_chosen(service: AuthService, *, password: str, **kwa _provision_steps = itertools.count(1) -def provision_totp() -> dict[str, Any]: +def provision_totp(secret: str = PROVISION_TOTP_SECRET) -> dict[str, Any]: """The TOTP arguments ``provision_first_administrator`` needs under the shipped ``require_mfa`` (ADR 0197 Amendment A): the secret, a code for it, and the instant that code was "read". Each call lands on its own later 30-second step, so repeated provisions of one row - (the repair tests) never replay a step the row already consumed.""" + (the repair tests) never replay a step the row already consumed. ``secret`` is for a test + whose runs must show different entries.""" from messagefoundry.auth import totp at = time.time() + totp.DEFAULT_PERIOD * next(_provision_steps) return { - "totp_secret": PROVISION_TOTP_SECRET, - "totp_code": totp.totp(PROVISION_TOTP_SECRET, now=at), + "totp_secret": secret, + "totp_code": totp.totp(secret, now=at), "totp_code_read_at": at, } diff --git a/tests/test_mfa.py b/tests/test_mfa.py index 8b2eb0c907..92a43c5f45 100644 --- a/tests/test_mfa.py +++ b/tests/test_mfa.py @@ -18,6 +18,8 @@ import asyncio import heapq import inspect +import json +import logging import time import unicodedata from collections.abc import Awaitable @@ -2485,7 +2487,12 @@ async def reset_then_enable( rows = await store.list_audit( actor=ADMIN_USERNAME, action="auth.mfa_enroll_refused", limit=10 ) - assert len(rows) == 1 and '"reason": "secret_cleared"' in str(rows[0]["detail"]) + # ``session_ended`` is read from the revoke's outcome, not a constant (vault BACKLOG #3113). + assert len(rows) == 1 + assert json.loads(str(rows[0]["detail"])) == { + "reason": "secret_cleared", + "session_ended": True, + } actions = [e["action"] for e in await service.security_events_for(ADMIN_USERNAME)] assert "auth.mfa_enrolled" not in actions finally: @@ -2564,6 +2571,50 @@ async def other_confirm_first( await store.close() +async def test_a_refused_enable_whose_revoke_raises_still_writes_its_refusal_row( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """RED when: a store refusal of the revoke skips the ``auth.mfa_enroll_refused`` row (vault + BACKLOG #3113 item 2), or the row claims the session ended when it did not (item 7).""" + store = await _store() + try: + service = AuthService(store, AuthSettings(mfa_verify_min_elapsed_seconds=0)) + identity, token, _ = await login_admin(service) + enroll = await service.begin_mfa_enrollment(identity) + + real_enable = store.enable_totp + + async def reset_then_enable( + user_id: str, *, recovery_code_hashes: list[str], now: float | None = None + ) -> bool: + await store.disable_totp(user_id) + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes, now=now) + + async def refused_revoke(token_hash: str, *, now: float | None = None) -> None: + raise RuntimeError("the store refused the revoke") + + monkeypatch.setattr(store, "enable_totp", reset_then_enable) + monkeypatch.setattr(store, "revoke_session", refused_revoke) + with caplog.at_level(logging.ERROR, logger="messagefoundry.auth.service"): + refused = await service.confirm_mfa_enrollment( + identity, fresh_totp(enroll.secret), token=token + ) + # Fails closed all the same: no token and no codes come back. + assert refused.ok is False and refused.session_lost is True + assert refused.token is None and refused.recovery_codes == () + rows = await store.list_audit( + actor=ADMIN_USERNAME, action="auth.mfa_enroll_refused", limit=10 + ) + assert len(rows) == 1, "the refusal row was skipped when the revoke raised" + detail = json.loads(str(rows[0]["detail"])) + assert detail == {"reason": "secret_cleared", "session_ended": False} + assert any( + "could not end the session it rotated" in r.getMessage() for r in caplog.records + ), "the refused revoke was swallowed without a log line" + finally: + await store.close() + + # --- ADR 0197 Amendment A, wave 1 (BACKLOG #1131): every account is born with a way past ---------- # # A lock exists to bound guesses. On an engine-generated 192-bit credential it bounds nothing, so diff --git a/tests/test_provision_first_administrator.py b/tests/test_provision_first_administrator.py index 5e0b88e4a7..bc826e98d2 100644 --- a/tests/test_provision_first_administrator.py +++ b/tests/test_provision_first_administrator.py @@ -13,12 +13,14 @@ from __future__ import annotations import asyncio +import base64 import io import json import sys import time import types from collections.abc import Callable, Sequence +from contextvars import ContextVar from pathlib import Path import pytest @@ -2708,3 +2710,69 @@ def test_the_last_wrong_code_is_not_told_to_try_again( != 0 ) assert sum("did not match" in text for text in terminal.shown) == 4 + + +# --- vault BACKLOG #3113 item 1: the run that loses an enrolment race -------------------------- + +#: A second synthetic base32 secret, so the two runs show different entries. Encoded here rather +#: than written as a literal, which the gitleaks generic-api-key rule reads as a credential. +_OTHER_TOTP_SECRET = base64.b32encode(b"1234567890").decode("ascii") + + +async def test_the_run_that_loses_an_enrolment_race_is_not_told_to_remove_a_live_entry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Two runs repair one roleless row. The second stages its secret after the first, so the + first run's enable turns on the SECOND run's secret, and the second run's enable is refused. + Its refusal must not tell its operator to remove the entry it showed: that entry is the live + one. This is the re-stage race of vault BACKLOG #3025, driven in a fixed order.""" + store = await MessageStore.open(":memory:") + try: + service = AuthService(store, AuthSettings()) + await service.initialize() + await _roleless_account(store, email=None) + first = provision_totp() + second = provision_totp(_OTHER_TOTP_SECRET) + real_enable = store.enable_totp + second_at_enable = asyncio.Event() + first_enabled = asyncio.Event() + run: ContextVar[str] = ContextVar("run", default="first") + tasks: list[asyncio.Task[object]] = [] + + async def run_second() -> object: + run.set("second") + return await service.provision_first_administrator( + username="site-admin", password=_PASSWORD, actor="second", **second + ) + + async def racing_enable(user_id: str, *, recovery_code_hashes: list[str]) -> bool: + if run.get() == "first": + # The first run has staged its secret. The second run now stages its own over it, + # and stops at its own enable. + tasks.append(asyncio.create_task(run_second())) + await second_at_enable.wait() + try: + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes) + finally: + first_enabled.set() + second_at_enable.set() + await first_enabled.wait() + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes) + + monkeypatch.setattr(store, "enable_totp", racing_enable) + won = await service.provision_first_administrator( + username="site-admin", password=_PASSWORD, actor="first", **first + ) + assert won.repaired + with pytest.raises(FirstAdministratorRefused) as lost: + await tasks[0] + # The premise, measured: the secret the losing run showed is the one turned on. + row = await store.get_user("roleless") + assert row is not None and row.totp_enabled + assert await store.get_totp_secret("roleless") == _OTHER_TOTP_SECRET + message = str(lost.value) + assert "did not turn TOTP on and granted no role" in message + assert "remove" not in message.lower() + assert "keep the authenticator entry it showed" in message + finally: + await store.close() From d2c88da1aea5971dbc02277293fc10498624dbc0 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 16:49:07 -0500 Subject: [PATCH 04/27] Review round one on the D1 auth follow-ups (BACKLOG #3164 #3121 #3113) #3121: the directory line "not from the MessageFoundry console" is gone from the role re-sync and disable notices. A re-sync's roles come from the directory's groups through the group-to-role map an administrator edits in the console, so that line could be false. The role notice now says MessageFoundry changed the roles to match the directory's groups. from_directory is gated on the kind again, so the reconciler's directory_sessions_ended, which now carries the source tag, does not claim the directory changed anything. A directory ACCOUNT_DISABLED gets its own subject, since the administrator subject says the MessageFoundry account was disabled. Tests render the as-sent shapes and pin the subject. Comments that called a directory change an administrator's act are reworded. #3113: the race test cancels the second run on failure and uses a low-entropy synthetic secret; the PHASE0 row links SECURITY.md rather than restating it; the revoke comment states it covers a refusal of the revoke alone. #3164: the lifespan comment names what the loop is already running. Proposed PR title: Auth follow-ups: start-up probe off the loop, directory notice wording, TOTP enrolment refusal fixes (BACKLOG #3164 #3121 #3113) Proposed banner, #3164: BUILT on engine branch b257-d1. The lifespan awaits asyncio.to_thread(auth.probe_credential_generation) with its guard kept; tests/test_credential_probe_off_loop.py proves the generator runs off the loop thread. Proposed banner, #3121: BUILT on engine branch b257-d1. Reconciler and sign-in re-sync ROLES_CHANGED and ACCOUNT_DISABLED notices carry source directory, name no administrator, and close with "If you did not expect this change"; the directory disable has its own subject; administrator wording kept; tests pin descriptions and subject. Proposed banner, #3113: BUILT on engine branch b257-d1. Items 1 to 7 done with tests (race test for item 1, raise test for items 2 and 7); items 8 and 9 declined in writing in confirm_mfa_enrollment. The PHASE0 line is corrected; no vault docs/security file quotes it, so no cell re-read is owed unless the scorecard holder finds one. --- docs/ASVS-L2-PHASE0-CHANGES.md | 2 +- docs/SECURITY.md | 6 ++- messagefoundry/api/app.py | 3 +- messagefoundry/auth/notifications.py | 5 +- messagefoundry/auth/service.py | 7 ++- messagefoundry/pipeline/security_notify.py | 56 ++++++++++++++------ tests/test_provision_first_administrator.py | 13 +++-- tests/test_security_notify.py | 58 ++++++++++++++------- 8 files changed, 103 insertions(+), 47 deletions(-) diff --git a/docs/ASVS-L2-PHASE0-CHANGES.md b/docs/ASVS-L2-PHASE0-CHANGES.md index 4bcdd77e74..a5eb73d18e 100644 --- a/docs/ASVS-L2-PHASE0-CHANGES.md +++ b/docs/ASVS-L2-PHASE0-CHANGES.md @@ -468,7 +468,7 @@ is right. | Off-box log-forward client key | 4, on the same condition as the API TLS server key | Loads the combined PEM in `_build_tls_context` ([`logging_setup.py`](../messagefoundry/logging_setup.py)). There is no passphrase setting | As for the API TLS server key. The file's permissions are the key's only protection, so setting them is the site's job | | Native API client key | 4, on the same condition as the API TLS server key | Holds no copy. The client process loads it (`_build_verify_context` in [`apiclient/client.py`](../messagefoundry/apiclient/client.py)) | Every function in this row belongs to the operator of the client machine | | Non-production self-signed key | Two keys, each 4, private authentication key, like the API TLS placeholder key. The `cert self-signed` key is a placeholder a site may serve. The load-test harness pair is also test keying material under section 8.1.4 | Mints both with `make_self_signed` in [`pki.py`](../messagefoundry/pki.py), the CLI key on request. The CLI writes its key through `_write_private_key`, which refuses to overwrite an existing file. The harness writes its pair with a plain write and `chmod`, as the lifecycle row above says. Nothing is audited | The `cert self-signed` key is a placeholder key, like the API TLS placeholder key: a site may serve it until it supplies a chain. Set its cryptoperiod under section 5.3.6 item 4 through `--days`, because the engine never renews it. Replace it with an operator-supplied chain, then delete both files. The load-test harness pair is test keying material, which section 8.1.4 says shall not be used operationally, and the engine does not stop that | -| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable itself checks only that some secret is still staged and TOTP is still off, not that the staged secret is the one the code proved, so a second enrolment start that re-stages the secret inside the confirm's window can turn on a secret no code proved (vault BACKLOG #3025, open; corrected under vault BACKLOG #3113). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | +| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable does not check that the staged secret is the one the code proved; [`SECURITY.md`](SECURITY.md), in its TOTP paragraph, states that gap (vault BACKLOG #3025; this sentence corrected under vault BACKLOG #3113). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | | Anonymizer re-identification salt | None of the 19 types in 5.1.1 fits. It keys a hash that pseudonymizes one dataset. The nearest fit is 5.1.2's *other secret information*, so this row borrows 5.3.7's rule for that and takes no cryptoperiod from 5.3.6 | Never mints or stores it. `Keyer` in [`anon/keying.py`](../messagefoundry/anon/keying.py) refuses a visibly weak salt, and the export reads it from the environment for one run | Mint it with a CSPRNG, one per dataset, and hold it with one person. Section 5.3.7 says other secret information shall not be kept longer than necessary, so its period is one dataset. Keep no backup unless the dataset must be regenerated, and destroy it when that need ends | | Sealed-cache key | 6, symmetric data-encryption key. It protects the confidentiality and integrity of decoded cache values in the engine's own memory | Mints it per process (`_Sealer` in [`store/sealed_cache.py`](../messagefoundry/store/sealed_cache.py)) and uses it only to seal and open cache values, with each value bound to its cache key as associated data. Pre-activation and active are one step, and the key is destroyed with the process, so its cryptoperiod (5.3.6) is the process lifetime. There is no backup or archive of it (Table 7 is moot) | Protect the engine process's memory: the in-use residual in `docs/PHI.md` §3 and the host requirements in §10 apply, because this key sits in the same memory as the values it seals | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index cd9bb91528..f3fd6c4198 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4387,11 +4387,13 @@ Users are notified of security-relevant changes to their account through **two** failed attempts**, or a step-up re-auth that clears such a run (suspicious-login signals, 6.3.5); and **password change**, **email change**, **role change**, **account disable**, and a **directory rename** of the username (credential changes, 6.3.7). - A notice for a change the directory made names the directory and no administrator, and it closes + A notice for a change the directory drove names the directory and no administrator, and it closes with "If you did not expect this change" rather than "If this was you". That covers at least the directory rename, the directory's email change, a role change applied by the session reconciler or by a directory sign-in's re-sync, and the reconciler's `account_disabled` when it read the - account disabled (vault BACKLOG #3121). When the reconciler ends an account's sessions because it + account disabled (vault BACKLOG #3121). The role notice says the roles were changed to match the + directory's groups, not that the console played no part: those groups map to roles through the + group-to-role map an administrator edits. When the reconciler ends an account's sessions because it could not match the account or could not read its `userAccountControl`, it sends `directory_sessions_ended`, not `account_disabled` (vault BACKLOG #2140). The rename notice, `username_changed`, names the old and the new name, and is sent only when diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 3fe2b485f8..6370ef36ed 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -10491,7 +10491,8 @@ async def _audit_upload_prune(meta: UploadedFileMeta) -> None: # an ERROR and returns, and is guarded like the census: a policy-level fact must # never refuse the start, whatever a later policy clause raises. It runs off the loop # (vault BACKLOG #3164): a pathological context-word list makes the generator retry - # its policy screens many times, and the loop already serves the API by now. + # its policy screens many times, and the engine's listeners and workers already run + # on the loop by now. try: await asyncio.to_thread(auth.probe_credential_generation) except Exception: # noqa: BLE001 -- see the comment above diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index 9d108998b4..015e50cfc0 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -50,8 +50,9 @@ TEMPORARY_CREDENTIAL_EXPIRING = "temporary_credential_expiring" TEMPORARY_CREDENTIAL_EXPIRING_FOR_ISSUER = "temporary_credential_expiring_issuer" EMAIL_CHANGED = "email_changed" # 6.3.7 — the account's email address was changed -# 6.3.7 -- the account's roles changed: an administrator's act, or the directory's, which the -# reconciler and the sign-in re-sync tag with ``detail["source"] == "directory"`` (vault BACKLOG #3121). +# 6.3.7 -- the account's roles changed: an administrator's act, or a re-sync from the directory's +# groups, which the reconciler and the sign-in re-sync tag ``detail["source"] == "directory"`` +# (vault BACKLOG #3121). ROLES_CHANGED = "roles_changed" # 6.3.7 -- the directory renamed the account and the engine copied the new name onto the row # (BACKLOG #2017). When it is sent is stated once, at the send site in diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 485a2b6cab..279c57b0e3 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -5459,7 +5459,7 @@ async def _complete_ad_login( # local one, so notify the affected user out-of-band too (ASVS 6.3.7), matching set_roles(). # Best-effort; the change is also visible in the audited /me/security-events feed. # ``source`` words it for the directory, as the reconciler's notice is (vault BACKLOG - # #3121): no administrator made this change, so "if this was you" cannot apply. + # #3121): the holder did not make this change, so "if this was you" cannot apply. await self._notify_security( ROLES_CHANGED, username=user.username, @@ -10159,7 +10159,10 @@ async def confirm_mfa_enrollment( # # A store refusal of the revoke is logged, not raised, so the refusal row below is # still written (vault BACKLOG #3113). ``session_ended`` records what the revoke did. - # The token is not handed back either way, so the caller still cannot use the session. + # The token is not handed back either way, so the caller still cannot use the session; + # a session the revoke left live expires on its own timers. A fault that also refuses + # the read and the audit write below still loses the row: this covers a refusal of the + # revoke alone. rotated = hash_token(elevation.token) session_ended = await write_audit_soft( lambda: self._store.revoke_session(rotated), diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index 9c82556f3d..e62f795d78 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -166,18 +166,42 @@ ), } -# Vault BACKLOG #3121. A directory-driven role change or disable names no administrator: the -# directory made it. The directory line and the "If you did not expect this change" closing follow -# from ``from_directory`` in :func:`_build_body`, so these say only what changed. The disable reads -# what the reconciler did, which is end the sessions; it does not claim MessageFoundry disabled -# anything. +# Vault BACKLOG #3121. The reconciler's and the sign-in re-sync's role change and disable name no +# administrator and close with "If you did not expect this change". Each carries its own source in +# its words, so neither takes the generic directory line: a re-sync's roles come from the +# directory's groups through the group-to-role map, which an administrator edits in the console, +# so "not from the MessageFoundry console" would be false for it. The disable says what the +# reconciler did, which is end the sessions; no MessageFoundry row was disabled. _DIRECTORY_DESCRIPTIONS = { - ROLES_CHANGED: "Your account's roles were changed.", + ROLES_CHANGED: ( + "MessageFoundry read your groups in your organization's directory and changed your " + "account's roles to match them." + ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " "your sessions." ), } +# The administrator subject says the MessageFoundry account was disabled, which is false here. +_DIRECTORY_SUBJECTS = { + ACCOUNT_DISABLED: "Your MessageFoundry sessions were ended: your directory account is disabled", +} + + +def _from_directory(event: SecurityEvent) -> bool: + """Whether the directory, not the holder or an administrator, made this change (BACKLOG #1139, + #2291, vault #3121). Gated on the kind as well as the tag: the reconciler tags every notice it + sends, and its ``directory_sessions_ended`` must not claim the directory changed anything.""" + return ( + event.event_type in (EMAIL_CHANGED, USERNAME_CHANGED, *_DIRECTORY_DESCRIPTIONS) + and event.detail.get("source") == "directory" + ) + + +def _subject(event: SecurityEvent) -> str: + if _from_directory(event) and event.event_type in _DIRECTORY_SUBJECTS: + return _DIRECTORY_SUBJECTS[event.event_type] + return _SUBJECTS.get(event.event_type, "MessageFoundry security alert") _LOCAL_REMEDY = ( @@ -269,10 +293,7 @@ def _build_body(event: SecurityEvent) -> str: set_by_admin = ( event.event_type == NOTIFY_EMAIL_SET and event.detail.get("set_by") == "administrator" ) - # BACKLOG #1139, #2291: the directory made this change, not the holder and not the console. - # The sender's tag decides, whatever the kind (vault BACKLOG #3121), so a new directory notice - # cannot fall back to the administrator wording by missing from a list here. - from_directory = event.detail.get("source") == "directory" + from_directory = _from_directory(event) if moved_by_admin: description = ( "An administrator changed the address that receives security notices for your account." @@ -282,7 +303,6 @@ def _build_body(event: SecurityEvent) -> str: "An administrator set this address to receive security notices about your account." ) elif from_directory and event.event_type in _DIRECTORY_DESCRIPTIONS: - # Only these two kinds name an administrator in their own text; the others need no swap. description = _DIRECTORY_DESCRIPTIONS[event.event_type] else: description = _DESCRIPTIONS.get( @@ -389,11 +409,12 @@ def _build_body(event: SecurityEvent) -> str: scrub_log_argument(event.username), key, ) - if from_directory: + if from_directory and event.event_type not in _DIRECTORY_DESCRIPTIONS: # BACKLOG #1139. Say WHERE the change came from, because it changes what the reader can do # about it: a directory-driven change is not editable in the console, so "contact your # administrator" is the only action, and an unexplained change the holder cannot find a - # cause for reads as a compromise. + # cause for reads as a compromise. The vault #3121 kinds name their source in their own + # description instead (see ``_DIRECTORY_DESCRIPTIONS``). lines.append( "This change came from your organization's directory, not from the MessageFoundry " "console." @@ -499,9 +520,10 @@ def _build_body(event: SecurityEvent) -> str: DIRECTORY_SESSIONS_ENDED, ) ): - # A directory rename or email change is an administrator's act, so "if this was you" - # cannot apply to it (the email half is BACKLOG #2291), nor to a directory role change or - # disable (vault BACKLOG #3121). Nor to a federated link or unlink: + # The holder did not make a directory change, so "if this was you" cannot apply to a + # directory rename or email change (the email half is BACKLOG #2291), nor to a role + # change or disable the directory drove (vault BACKLOG #3121). Nor to a federated link or + # unlink: # both routes refuse an administrator changing their own account's binding. Nor to the # engine ending sessions on a directory answer (vault BACKLOG #2140). closing = "If you did not expect this change, contact your MessageFoundry administrator." @@ -677,7 +699,7 @@ def _send(self, event: SecurityEvent) -> None: port=self._port, sender=self._sender, recipients=[event.email], - subject=_SUBJECTS.get(event.event_type, "MessageFoundry security alert"), + subject=_subject(event), body=_build_body(event), use_tls=self._use_tls, username=self._username, diff --git a/tests/test_provision_first_administrator.py b/tests/test_provision_first_administrator.py index bc826e98d2..345f82a79a 100644 --- a/tests/test_provision_first_administrator.py +++ b/tests/test_provision_first_administrator.py @@ -13,7 +13,6 @@ from __future__ import annotations import asyncio -import base64 import io import json import sys @@ -2714,9 +2713,9 @@ def test_the_last_wrong_code_is_not_told_to_try_again( # --- vault BACKLOG #3113 item 1: the run that loses an enrolment race -------------------------- -#: A second synthetic base32 secret, so the two runs show different entries. Encoded here rather -#: than written as a literal, which the gitleaks generic-api-key rule reads as a credential. -_OTHER_TOTP_SECRET = base64.b32encode(b"1234567890").decode("ascii") +#: A second synthetic base32 secret, so the two runs show different entries. Two symbols on +#: purpose, for the reason ``_PASSWORD`` above is low-entropy: it must not look like a secret. +_OTHER_TOTP_SECRET = "ABABABABABABABAB" async def test_the_run_that_loses_an_enrolment_race_is_not_told_to_remove_a_live_entry( @@ -2770,9 +2769,15 @@ async def racing_enable(user_id: str, *, recovery_code_hashes: list[str]) -> boo row = await store.get_user("roleless") assert row is not None and row.totp_enabled assert await store.get_totp_secret("roleless") == _OTHER_TOTP_SECRET + # The winning run reported success over a secret it did not show. That is the race + # itself, vault BACKLOG #3025, pinned here as the premise and not as a wanted outcome. message = str(lost.value) assert "did not turn TOTP on and granted no role" in message assert "remove" not in message.lower() assert "keep the authenticator entry it showed" in message finally: + # A failure above must not leave the second run running against a closed store. + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) await store.close() diff --git a/tests/test_security_notify.py b/tests/test_security_notify.py index beea972807..7572319bae 100644 --- a/tests/test_security_notify.py +++ b/tests/test_security_notify.py @@ -37,6 +37,7 @@ _SUBJECTS, SecurityEventNotifier, _build_body, + _subject, security_notifier_from_settings, ) @@ -204,19 +205,38 @@ def test_a_non_last_factor_removal_renders_its_own_subject_and_body() -> None: assert "disabled" not in body.lower() -def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_disable() -> None: +#: The generic line a directory email change or rename carries (BACKLOG #1139). +_DIRECTORY_LINE = ( + "This change came from your organization's directory, not from the MessageFoundry console." +) + + +@pytest.mark.parametrize( + "detail", + [ + pytest.param({}, id="untagged"), + # The shape the reconciler mails since vault BACKLOG #3121. + pytest.param({"reason": "directory_absent", "source": "directory"}, id="as-sent"), + ], +) +def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_disable( + detail: dict[str, Any], +) -> None: """vault BACKLOG #2140. Both renderers fall back silently, so a half-wired kind would mail the generic wording. The directory never answered that the account is disabled or gone, so the text - must not say either, and an engine act cannot carry the "if this was you" closing.""" + must not say either, nor that the directory changed anything, and an engine act cannot carry + the "if this was you" closing.""" assert DIRECTORY_SESSIONS_ENDED in _SUBJECTS - body = _build_body(SecurityEvent(DIRECTORY_SESSIONS_ENDED, username="jdoe", email=None)) - text = (_SUBJECTS[DIRECTORY_SESSIONS_ENDED] + "\n" + body).lower() + event = SecurityEvent(DIRECTORY_SESSIONS_ENDED, username="jdoe", email=None, detail=detail) + body = _build_body(event) + text = (_subject(event) + "\n" + body).lower() assert "a security event occurred on your account." not in text # The description itself, not words the subject alone would satisfy (vault BACKLOG #3121). assert ( "A check against your organization's directory ended your MessageFoundry sessions." in body.splitlines() ) + assert _DIRECTORY_LINE not in body.splitlines() for claim in ("disabled", "deleted", "if this was you"): assert claim not in text assert body.endswith( @@ -226,15 +246,15 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di #: vault BACKLOG #3121. The description each directory-driven kind carries, pinned whole. _DIRECTORY_TEXT = { - ROLES_CHANGED: "Your account's roles were changed.", + ROLES_CHANGED: ( + "MessageFoundry read your groups in your organization's directory and changed your " + "account's roles to match them." + ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " "your sessions." ), } -_DIRECTORY_LINE = ( - "This change came from your organization's directory, not from the MessageFoundry console." -) _DIRECTORY_CLOSING = "If you did not expect this change, contact your MessageFoundry administrator." _ADMIN_TEXT = { ROLES_CHANGED: "Your account's roles were changed by an administrator.", @@ -250,22 +270,24 @@ def test_a_directory_role_or_disable_notice_names_the_directory_and_no_administr kind: str, ) -> None: """vault BACKLOG #3121. The reconciler and the sign-in re-sync tag these ``source`` directory: - no administrator made the change and the holder did nothing, so neither the administrator - wording nor "if this was you" may appear.""" - body = _build_body( - SecurityEvent( - kind, - username="jdoe", - email="j@example.org", - detail={"reason": "directory_disabled", "source": "directory"}, - ) + the holder did nothing, so neither the administrator wording nor "if this was you" may appear. + Nor the generic directory line: a role re-sync runs through the group-to-role map an + administrator edits in the console, so "not from the MessageFoundry console" could be false.""" + event = SecurityEvent( + kind, + username="jdoe", + email="j@example.org", + detail={"reason": "directory_disabled", "source": "directory"}, ) + body = _build_body(event) lines = body.splitlines() assert _DIRECTORY_TEXT[kind] in lines - assert _DIRECTORY_LINE in lines + assert _DIRECTORY_LINE not in lines assert _ADMIN_TEXT[kind] not in lines assert "if this was you" not in body.lower() assert body.endswith(_DIRECTORY_CLOSING) + # The administrator subject says the MessageFoundry account was disabled, which is false here. + assert "was disabled" not in _subject(event) @pytest.mark.parametrize("kind", [ROLES_CHANGED, ACCOUNT_DISABLED]) From 1d5ff351a13851e093d97381d4ef4343aca46db4 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:14:59 -0500 Subject: [PATCH 05/27] D1 round-two follow-ups: the enrolment refusal row survives any revoke fault (BACKLOG #3113) Round-two findings 1, 3, 5 and 6 from the D1 review, carried to this branch. - Finding 1: confirm_mfa_enrollment writes auth.mfa_enroll_refused in a finally around the revoke, so a defect the soft write re-raises (NotImplementedError, sqlite3.ProgrammingError) no longer skips the row, and aiosqlite's closed-connection ValueError is logged as a store refusal, so the confirm still answers session_lost. The row's read and write move into _audit_enroll_refused. - Finding 3: the directory role notice says the sessions already open on the account were ended, as the disable notice does; both senders (reconciler pass, sign-in re-sync) revoke first. - Finding 5: the DIRECTORY_SESSIONS_ENDED comment in auth/notifications.py no longer says ACCOUNT_DISABLED names an administrator. - Finding 6: docs/SECURITY.md says the directory notice names no administrator as what made the change (its closing still names one), and no longer says flatly that the refused confirm ends the session. Findings 2, 4 and 7 stay open by brief. --- docs/SECURITY.md | 17 +++-- messagefoundry/auth/notifications.py | 4 +- messagefoundry/auth/service.py | 74 +++++++++++++--------- messagefoundry/pipeline/security_notify.py | 8 ++- tests/test_mfa.py | 52 ++++++++++++++- tests/test_security_notify.py | 4 +- 6 files changed, 112 insertions(+), 47 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index f3fd6c4198..9aa5de2ebc 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2769,12 +2769,13 @@ An administrator clears a lost authenticator via `POST /users/{id}/reset-mfa` (which also revokes the user's sessions). A confirm turns TOTP on only while a secret is still staged and TOTP is still off (BACKLOG #2224). A reset can clear the secret during the confirm, or another confirm can turn TOTP on -first. Either way the confirm turns nothing on and returns no recovery codes. It ends the session -it just rotated and answers as a lost session: `401` on `POST /me/mfa/confirm`, and the sign-in -page in the web console. It is audited `auth.mfa_enroll_refused`. Its `reason` is +first. Either way the confirm turns nothing on and returns no recovery codes. It asks the store to +end the session it just rotated, and answers as a lost session: `401` on `POST /me/mfa/confirm`, +and the sign-in page in the web console. It is audited `auth.mfa_enroll_refused`. Its `reason` is `secret_cleared`, `already_enabled` or `account_gone`. That reason is read after the refusal, so a later write can blur it. Its `session_ended` is false when the store refused to end the rotated -session; the token is still not handed back (vault BACKLOG #3113). The check is that some secret is +session. The token is still not handed back, and the row is written whatever the revoke raised +(vault BACKLOG #3113). The check is that some secret is staged, not that it is the one the confirm verified: a second enrolment start that re-stages the secret inside the same window is not caught by it. With `[security].require_mfa` on — **the default since BACKLOG #187 (secure-by-default, including the @@ -4387,13 +4388,15 @@ Users are notified of security-relevant changes to their account through **two** failed attempts**, or a step-up re-auth that clears such a run (suspicious-login signals, 6.3.5); and **password change**, **email change**, **role change**, **account disable**, and a **directory rename** of the username (credential changes, 6.3.7). - A notice for a change the directory drove names the directory and no administrator, and it closes - with "If you did not expect this change" rather than "If this was you". That covers at least the + A notice for a change the directory drove names the directory, not an administrator, as what made + the change. It closes with "If you did not expect this change, contact your MessageFoundry + administrator" rather than "If this was you". That covers at least the directory rename, the directory's email change, a role change applied by the session reconciler or by a directory sign-in's re-sync, and the reconciler's `account_disabled` when it read the account disabled (vault BACKLOG #3121). The role notice says the roles were changed to match the directory's groups, not that the console played no part: those groups map to roles through the - group-to-role map an administrator edits. When the reconciler ends an account's sessions because it + group-to-role map an administrator edits. Like the disable notice, it says the sessions already + open on the account were ended, which both of its senders do first. When the reconciler ends an account's sessions because it could not match the account or could not read its `userAccountControl`, it sends `directory_sessions_ended`, not `account_disabled` (vault BACKLOG #2140). The rename notice, `username_changed`, names the old and the new name, and is sent only when diff --git a/messagefoundry/auth/notifications.py b/messagefoundry/auth/notifications.py index 015e50cfc0..0a7015e8dc 100644 --- a/messagefoundry/auth/notifications.py +++ b/messagefoundry/auth/notifications.py @@ -76,8 +76,8 @@ # 6.3.7 -- the directory session reconciler ended the account's sessions for a reason that is NOT a # read disabled bit: the probe could not match the account (``directory_absent``: not found, or its # stored id did not match or could not be parsed), or its ``userAccountControl`` could not be read -# (vault BACKLOG #2140). ACCOUNT_DISABLED would tell the holder an administrator disabled the account, -# which nothing established. Which reason sends which kind is decided once, in +# (vault BACKLOG #2140). ACCOUNT_DISABLED would tell the holder the account is disabled, by an +# administrator or in the directory, which nothing established. Which reason sends which kind is decided once, in # ``AuthService._apply_reconcile_revocation``. DIRECTORY_SESSIONS_ENDED = "directory_sessions_ended" MFA_ENABLED = "mfa_enabled" # 6.3.7 — a second factor (TOTP) was enrolled on the account diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 279c57b0e3..4b6eb9b71c 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -10157,38 +10157,27 @@ async def confirm_mfa_enrollment( # ended. The order is #1902's, rotate first and enable last, and is kept. Vault BACKLOG # #3113 item 8 is declined for that reason: enabling first would undo #1902. # - # A store refusal of the revoke is logged, not raised, so the refusal row below is - # still written (vault BACKLOG #3113). ``session_ended`` records what the revoke did. - # The token is not handed back either way, so the caller still cannot use the session; - # a session the revoke left live expires on its own timers. A fault that also refuses - # the read and the audit write below still loses the row: this covers a refusal of the - # revoke alone. + # A store refusal of the revoke is logged, not raised, so the confirm still answers + # ``session_lost`` (vault BACKLOG #3113). That covers aiosqlite's closed connection + # too, which it raises as a bare ``ValueError`` before sqlite3 is reached. A defect + # the helper re-raises still propagates, and the ``finally`` writes the refusal row + # first, so no exception the revoke raises can skip it. ``session_ended`` records + # what the revoke did. The token is not handed back either way, so the caller still + # cannot use the session; a session the revoke left live expires on its own timers. + # A fault that also refuses the read and the audit write below still loses the row. rotated = hash_token(elevation.token) - session_ended = await write_audit_soft( - lambda: self._store.revoke_session(rotated), - log=_log, - message="a refused TOTP enrolment for %s could not end the session it rotated", - args=lambda: (scrub_log_argument(identity.username),), - errors=_audit_write_errors(), - defects=AUDIT_WRITE_DEFECTS, - ) - # The reason is the row's state read AFTER the refusal, not the write's own verdict, - # so a later write can blur it; it is a pointer for an operator, not a proof. Vault - # BACKLOG #3113 item 9 is declined: the write's own verdict would need ``enable_totp`` - # to return a reason on all three store backends. - after = await self._load_user(identity.user_id) - if after is None: - reason = "account_gone" - elif after.totp_enabled: - reason = "already_enabled" - else: - reason = "secret_cleared" - await self._audit( - "auth.mfa_enroll_refused", - actor=identity.username, - detail=_json({"reason": reason, "session_ended": session_ended}), - client=client, - ) + session_ended = False + try: + session_ended = await write_audit_soft( + lambda: self._store.revoke_session(rotated), + log=_log, + message="a refused TOTP enrolment for %s could not end the session it rotated", + args=lambda: (scrub_log_argument(identity.username),), + errors=(*_audit_write_errors(), ValueError), + defects=AUDIT_WRITE_DEFECTS, + ) + finally: + await self._audit_enroll_refused(identity, session_ended, client=client) return Elevation(session_lost=True) await self._audit("auth.mfa_enrolled", actor=identity.username, client=client) # ADR 0197 Amendment A: enrolment now comes BEFORE the first rotation, so whoever intercepts @@ -10208,6 +10197,29 @@ async def confirm_mfa_enrollment( await self._mark_login_address_known(identity.user_id, client, enrolment=user) return elevation + async def _audit_enroll_refused( + self, identity: Identity, session_ended: bool, *, client: str | None + ) -> None: + """Write the ``auth.mfa_enroll_refused`` row for a confirm whose enable matched no row. + + The reason is the row's state read AFTER the refusal, not the write's own verdict, so a + later write can blur it; it is a pointer for an operator, not a proof. Vault BACKLOG #3113 + item 9 is declined: the write's own verdict would need ``enable_totp`` to return a reason + on all three store backends.""" + after = await self._load_user(identity.user_id) + if after is None: + reason = "account_gone" + elif after.totp_enabled: + reason = "already_enabled" + else: + reason = "secret_cleared" + await self._audit( + "auth.mfa_enroll_refused", + actor=identity.username, + detail=_json({"reason": reason, "session_ended": session_ended}), + client=client, + ) + async def verify_mfa( self, token: str | None, code: str, *, client: str | None = None ) -> Elevation: diff --git a/messagefoundry/pipeline/security_notify.py b/messagefoundry/pipeline/security_notify.py index e62f795d78..e06fa1ecf2 100644 --- a/messagefoundry/pipeline/security_notify.py +++ b/messagefoundry/pipeline/security_notify.py @@ -171,11 +171,13 @@ # its words, so neither takes the generic directory line: a re-sync's roles come from the # directory's groups through the group-to-role map, which an administrator edits in the console, # so "not from the MessageFoundry console" would be false for it. The disable says what the -# reconciler did, which is end the sessions; no MessageFoundry row was disabled. +# reconciler did, which is end the sessions; no MessageFoundry row was disabled. Both senders +# of the role change end the sessions already open on the account first, the reconciler's pass +# and the sign-in re-sync alike, so the role notice says so as the disable notice does. _DIRECTORY_DESCRIPTIONS = { ROLES_CHANGED: ( - "MessageFoundry read your groups in your organization's directory and changed your " - "account's roles to match them." + "MessageFoundry read your groups in your organization's directory, changed your " + "account's roles to match them, and ended any sessions already open on your account." ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " diff --git a/tests/test_mfa.py b/tests/test_mfa.py index 92a43c5f45..0c4b56cb0b 100644 --- a/tests/test_mfa.py +++ b/tests/test_mfa.py @@ -2571,8 +2571,17 @@ async def other_confirm_first( await store.close() +@pytest.mark.parametrize( + "fault", + [ + RuntimeError("the store refused the revoke"), + # aiosqlite's own refusal of a closed connection, raised before sqlite3 is reached. + ValueError("Connection closed"), + ], + ids=["store-refusal", "closed-connection"], +) async def test_a_refused_enable_whose_revoke_raises_still_writes_its_refusal_row( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, fault: Exception ) -> None: """RED when: a store refusal of the revoke skips the ``auth.mfa_enroll_refused`` row (vault BACKLOG #3113 item 2), or the row claims the session ended when it did not (item 7).""" @@ -2591,7 +2600,7 @@ async def reset_then_enable( return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes, now=now) async def refused_revoke(token_hash: str, *, now: float | None = None) -> None: - raise RuntimeError("the store refused the revoke") + raise fault monkeypatch.setattr(store, "enable_totp", reset_then_enable) monkeypatch.setattr(store, "revoke_session", refused_revoke) @@ -2615,6 +2624,45 @@ async def refused_revoke(token_hash: str, *, now: float | None = None) -> None: await store.close() +async def test_a_refused_enable_whose_revoke_hits_a_defect_still_writes_its_refusal_row( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """RED when: a defect the revoke raises, which the soft write re-raises rather than logs, skips + the ``auth.mfa_enroll_refused`` row (vault BACKLOG #3113 item 2, round two). The defect still + propagates, so a bug is not passed over as a store refusal.""" + store = await _store() + try: + service = AuthService(store, AuthSettings(mfa_verify_min_elapsed_seconds=0)) + identity, token, _ = await login_admin(service) + enroll = await service.begin_mfa_enrollment(identity) + + real_enable = store.enable_totp + + async def reset_then_enable( + user_id: str, *, recovery_code_hashes: list[str], now: float | None = None + ) -> bool: + await store.disable_totp(user_id) + return await real_enable(user_id, recovery_code_hashes=recovery_code_hashes, now=now) + + async def broken_revoke(token_hash: str, *, now: float | None = None) -> None: + raise NotImplementedError("a backend without the revoke") + + monkeypatch.setattr(store, "enable_totp", reset_then_enable) + monkeypatch.setattr(store, "revoke_session", broken_revoke) + with pytest.raises(NotImplementedError): + await service.confirm_mfa_enrollment(identity, fresh_totp(enroll.secret), token=token) + rows = await store.list_audit( + actor=ADMIN_USERNAME, action="auth.mfa_enroll_refused", limit=10 + ) + assert len(rows) == 1, "the refusal row was skipped when the revoke raised a defect" + assert json.loads(str(rows[0]["detail"])) == { + "reason": "secret_cleared", + "session_ended": False, + } + finally: + await store.close() + + # --- ADR 0197 Amendment A, wave 1 (BACKLOG #1131): every account is born with a way past ---------- # # A lock exists to bound guesses. On an engine-generated 192-bit credential it bounds nothing, so diff --git a/tests/test_security_notify.py b/tests/test_security_notify.py index 7572319bae..f9c5a3a124 100644 --- a/tests/test_security_notify.py +++ b/tests/test_security_notify.py @@ -247,8 +247,8 @@ def test_a_directory_sessions_ended_notice_renders_its_own_text_and_claims_no_di #: vault BACKLOG #3121. The description each directory-driven kind carries, pinned whole. _DIRECTORY_TEXT = { ROLES_CHANGED: ( - "MessageFoundry read your groups in your organization's directory and changed your " - "account's roles to match them." + "MessageFoundry read your groups in your organization's directory, changed your " + "account's roles to match them, and ended any sessions already open on your account." ), ACCOUNT_DISABLED: ( "Your organization's directory reports your account as disabled, so MessageFoundry ended " From c67c178bd0f5b4fd6ab25b187b3a746dd2c21fdc Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:23:40 -0500 Subject: [PATCH 06/27] A self-service session revoke that ends nothing writes one audit row (BACKLOG #3260) Limb 2 of vault BACKLOG #3260, built on the owner ruling of 2026-10-16 (fleet wiki note 20261016T211331989Z-vfzmx7). AuthService.revoke_own_session writes auth.session_revoke_missed through write_audit_soft on a miss, with no defect re-raised, so the 404 and the console's revoke_missed notice stand whatever the audit write does. The detail is the id's first 12 characters and nothing else, so an unknown id and another account's id write the same row as well as getting the same answer. The method takes a required keyword client, which both planes pass (DELETE /me/sessions/{id} and /ui/account/sessions/{id}/revoke); the auth.session_revoked row now carries it too. The seam digest records names, not parameters, so it did not move; the _ui_seam history and the console changelog say so. SECURITY.md and a changelog fragment describe the row. Tests: tests/test_self_target_refusal_audit.py (JSON plane, unknown and foreign in one run, plus the success control) and packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py (console plane, unknown, foreign and an over-long plain-text id). --- .../3260-session-revoke-miss.security.md | 7 ++ docs/SECURITY.md | 9 ++- messagefoundry/api/_ui_seam.py | 5 ++ messagefoundry/api/auth_routes.py | 7 +- messagefoundry/auth/service.py | 32 ++++++++- messagefoundry_webconsole/routes/account.py | 7 +- .../messagefoundry-webconsole/CHANGELOG.md | 5 ++ .../test_ui_session_revoke_miss_audit.py | 67 +++++++++++++++++++ tests/test_self_target_refusal_audit.py | 61 +++++++++++++++++ 9 files changed, 192 insertions(+), 8 deletions(-) create mode 100644 changelog.d/3260-session-revoke-miss.security.md create mode 100644 packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py diff --git a/changelog.d/3260-session-revoke-miss.security.md b/changelog.d/3260-session-revoke-miss.security.md new file mode 100644 index 0000000000..d5c8cc27c9 --- /dev/null +++ b/changelog.d/3260-session-revoke-miss.security.md @@ -0,0 +1,7 @@ +- **A self-service revoke of one session that ends nothing is now audited.** `DELETE + /me/sessions/{session_id}` and the console's revoke button answer as before when the id names no + session or another account's. Each such miss now writes one `auth.session_revoke_missed` row. It + names the actor, the caller's address and the first 12 characters of the id, and nothing else, so + the row does not tell an unknown id from another account's any more than the answer does. Before, + the miss left no row of its own on either plane. The `auth.session_revoked` row now carries the + caller's address too. See [SECURITY.md](../docs/SECURITY.md). (`vault BACKLOG #3260`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 9aa5de2ebc..880d7999c9 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -752,8 +752,11 @@ AD group-map and scope-map saves and the custom-role edit and delete can still e sessions on the login window. The row names the actor, the route as `op`, the caller's stored id and the caller's address. The guard refuses when the path's spelling or the stored id matches the caller (vault BACKLOG #3259); `_refuse_if_self` in `api/auth_routes.py` says why the -stored id matters on the console. The `404` that `DELETE /me/sessions/{session_id}` answers for -another account's session still writes no row of its own. +stored id matters on the console. A self-service revoke of one session that ends nothing, the +`404` of `DELETE /me/sessions/{session_id}` and the console's "Nothing was revoked" notice, writes +one `auth.session_revoke_missed` row (vault BACKLOG #3260, limb 2). It names the actor, the +caller's address and the first 12 characters of the id, and nothing else: an unknown id and +another account's id get the same answer and the same row. **Delegated identity & admin device posture (#193 sibling; ASVS 13.2.1 / 13.3.2 / 8.4.2 — the delegation boundary).** Three controls whose enforcement is largely the deploying organization's to provide. @@ -5170,7 +5173,7 @@ Every authentication and authorization event is written to the durable `audit_lo user: `auth.login_success` / `auth.login_failed` / `auth.login_locked` / `auth.logout` / `auth.login_new_ip` / `auth.login_address_unevaluated` (the first-seen sign-in address, BACKLOG #288; on a directory sign-in the row's `mech` names the leg, `kerberos` or `oidc`) / -`auth.permission_denied` / `auth.channel_denied` / `auth.self_target_refused`, the 6.3.5 events `auth.account_locked` / +`auth.permission_denied` / `auth.channel_denied` / `auth.self_target_refused` / `auth.session_revoke_missed`, the 6.3.5 events `auth.account_locked` / `auth.login_after_failures`, the re-proof rows `auth.reauth` / `auth.password_change_failed`, plus `user.created` / `user.roles_changed` / `user.channel_scope_changed` / `user.channel_scope_change_refused` / `user.deleted`, `ad_group_map.updated` / `ad_group_scope_map.updated`, and `auth.ad_scope_resynced`. PHI access (viewing a raw message or displaying patient summaries) is recorded diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index 68233ecc21..27901b2e09 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -404,6 +404,11 @@ #: surface grew. An older console on this engine is refused at mount; a newer console on an older #: engine would call a method that does not exist. #: +#: Vault BACKLOG #3260, limb 2: ``AuthService.revoke_own_session`` takes a required keyword +#: ``client``, the address on its new ``auth.session_revoke_missed`` row and on its +#: ``auth.session_revoked`` row, and ``/ui/account/sessions/{id}/revoke`` passes it. The digest did +#: not move, for the same reason as #3259, and the same TypeError skew applies to that call. +#: #: The digest below covers the surface DISCOVERED from the console's own imports and uses, which is #: strictly larger than the five hand-maintained tuples it replaced -- those had drifted, and the #: proof is that commit 40a4d5d9 added a REQUIRED ``UploadedFileList.scope`` field the console renders diff --git a/messagefoundry/api/auth_routes.py b/messagefoundry/api/auth_routes.py index adaac6a2d9..8f2348bcc7 100644 --- a/messagefoundry/api/auth_routes.py +++ b/messagefoundry/api/auth_routes.py @@ -962,6 +962,7 @@ async def count(pin: float | None) -> int: @app.delete("/me/sessions/{session_id}", response_model=SimpleMessage) async def revoke_my_session( session_id: DigestId, + request: Request, service: AuthService = Depends(_service), # 7.5.2 (ASVS): terminating a session needs a fresh PASSWORD re-proof BOUND TO THIS ACTION # (BACKLOG #1149) — single-use, so the login-seeded window no longer satisfies it. Still the @@ -971,7 +972,11 @@ async def revoke_my_session( identity: Identity = Depends(require_reauth_only_action(STEP_UP_ACTION_SESSION_TERMINATE)), ) -> SimpleMessage: # Ownership-checked in the service: a 404 (not 403) avoids confirming another user's session id. - if not await service.revoke_own_session(identity, session_id, actor=identity.username): + # A miss writes one audit row there, the same for an unknown and a foreign id (vault + # BACKLOG #3260, limb 2), and the answer here is unchanged. + if not await service.revoke_own_session( + identity, session_id, actor=identity.username, client=client_ip(request) + ): raise HTTPException(status.HTTP_404_NOT_FOUND, "no such session") return SimpleMessage(detail="session revoked") diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 4b6eb9b71c..4335c5e8aa 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -2027,6 +2027,11 @@ def _idp_error_fields(exc: BaseException) -> dict[str, object]: #: current credential's is the newest, so a page this size is ample. _REMINDER_MARK_PAGE: Final = 50 +#: Vault BACKLOG #3260, limb 2 (owner ruling 2026-10-16): the row a self-service revoke of one +#: session writes when it ends nothing, because the id names no session or another account's. +#: One action for both, and the same detail, so the row cannot tell the two apart either. +SESSION_REVOKE_MISSED_ACTION: Final = "auth.session_revoke_missed" + #: Vault BACKLOG #3260: the row an admin route writes when it refuses the caller's own account. #: Its detail's ``op`` names the route, one of :data:`SelfTargetOp`. SELF_TARGET_REFUSED_ACTION: Final = "auth.self_target_refused" @@ -8479,18 +8484,41 @@ async def list_sessions(self, user_id: str) -> list[SessionRecord]: await self._session_seal_failed(row.token_hash) # see identity_for_token return kept - async def revoke_own_session(self, identity: Identity, session_id: str, *, actor: str) -> bool: + async def revoke_own_session( + self, identity: Identity, session_id: str, *, actor: str, client: str | None + ) -> bool: """Revoke one of ``identity``'s **own** sessions by id (its ``token_hash``). Returns ``False`` if the session doesn't exist or isn't the caller's — so the API answers 404 without revealing - or letting a user touch another's session. Audited.""" + or letting a user touch another's session. Audited. + + A miss writes :data:`SESSION_REVOKE_MISSED_ACTION` (vault BACKLOG #3260, limb 2; owner + ruling 2026-10-16). Its detail is the id's first 12 characters, as the success row's is, + and nothing else: an unknown id and another account's id write the same row, so the trail + records no ownership a caller could not already learn. The write is soft with no defect + re-raised, for the reason ``api.auth_routes._refuse_if_self`` gives: a raise would replace + the answer the caller is owed. ``client`` is required, as on + :meth:`audit_self_target_refused`, because both planes call this from a request.""" session = await self._load_session(session_id) if session is None or session.user_id != identity.user_id: + await write_audit_soft( + lambda: self._audit( + SESSION_REVOKE_MISSED_ACTION, + actor=actor, + detail=_json({"session": session_id[:12]}), + client=client, + ), + log=_log, + message="a missed session revoke by %s stands, but its audit row failed", + args=lambda: (scrub_log_argument(actor),), + defects=(), + ) return False await self._store.revoke_session(session_id) await self._audit( "auth.session_revoked", actor=actor, detail=_json({"scope": "self", "session": session_id[:12]}), + client=client, ) return True diff --git a/messagefoundry_webconsole/routes/account.py b/messagefoundry_webconsole/routes/account.py index 72a813fb50..b332980515 100644 --- a/messagefoundry_webconsole/routes/account.py +++ b/messagefoundry_webconsole/routes/account.py @@ -626,8 +626,11 @@ async def ui_revoke_session( # time the POST lands: the target may have completed MFA or a step-up on its own device in # between. Reporting "revoked" then would tell the operator a session had ended while it was # still live under its new id. An unknown id and a foreign id - # read the same, so the notice discloses nothing the JSON twin's 404 does not. - revoked = await service.revoke_own_session(identity, session_id, actor=identity.username) + # read the same, so the notice discloses nothing the JSON twin's 404 does not. A miss writes + # the same audit row the JSON twin's does, in the service (vault BACKLOG #3260, limb 2). + revoked = await service.revoke_own_session( + identity, session_id, actor=identity.username, client=client_ip(request) + ) outcome = "revoked" if revoked else "revoke_missed" return RedirectResponse(f"/ui/account/sessions?m={outcome}", status_code=303) diff --git a/packaging/messagefoundry-webconsole/CHANGELOG.md b/packaging/messagefoundry-webconsole/CHANGELOG.md index c1a4d85a85..04cf29289c 100644 --- a/packaging/messagefoundry-webconsole/CHANGELOG.md +++ b/packaging/messagefoundry-webconsole/CHANGELOG.md @@ -92,6 +92,11 @@ this line.** ### Security +- **A session revoke that ends nothing is audited, with your address.** The revoke button on + `/ui/account/sessions` now passes the request's address to the engine, which writes one + `auth.session_revoke_missed` row when the id names no session or another account's. The notice + is unchanged. `AuthService.revoke_own_session` takes a required keyword `client`, which the seam + digest does not record, so a skew raises TypeError on that call. (vault BACKLOG #3260) - **The audit page shows where each entry came from.** `/ui/audit` has a Client column, which shows the address the engine recorded for the entry, or a dash when no client was in scope. An operator tracing a run of refused sign-ins can now see their source on the page, not only in diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py new file mode 100644 index 0000000000..2bbfad27e1 --- /dev/null +++ b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""Vault BACKLOG #3260, limb 2, on the console plane: a session revoke that ends nothing is audited. + +Owner ruling 2026-10-16: one ``auth.session_revoke_missed`` row per miss, with the id cut to its +first 12 characters, and the answer unchanged and the same for an unknown id and another account's. +The console takes the id as plain text, so the cut also bounds what a caller can put in the row. +The JSON plane's twin is ``tests/test_self_target_refusal_audit.py``. +""" + +from __future__ import annotations + +import json + +import pytest +from _ui_clients import PW, SAME_ORIGIN, cookie_login, mint_bound_proof, provision, ui_client + +from messagefoundry.auth import Role +from messagefoundry.auth.service import AuthService +from messagefoundry.auth.tokens import hash_token +from messagefoundry.config.settings import AuthSettings +from messagefoundry.pipeline import Engine + + +@pytest.mark.parametrize( + "kind", + [ + "unknown", + "foreign", + # Plain text on this plane, so the row must cut it, not copy it. + "long", + ], +) +async def test_a_missed_console_revoke_writes_one_row_and_the_same_answer( + engine: Engine, kind: str +) -> None: + """RED when: the console's revoke of a session it cannot end writes no audit row of its own, or + its answer or row tells an unknown id from another account's (vault BACKLOG #3260, limb 2).""" + service = AuthService( + engine.store, AuthSettings(require_mfa=False, admin_write_min_interval_seconds=0) + ) + await service.initialize() + await provision(service, "a", [Role.VIEWER.value]) + await provision(service, "b", [Role.VIEWER.value]) + theirs = await service.login("b", PW) + assert theirs.token is not None + target = { + "unknown": "0" * 64, + "foreign": hash_token(theirs.token), + "long": "x" * 200, + }[kind] + path = f"/ui/account/sessions/{target}/revoke" + async with ui_client(engine, service) as c: + await cookie_login(c, "a") + await mint_bound_proof(c, path) + r = await c.post(path, headers=SAME_ORIGIN) + assert r.status_code == 303, (r.status_code, r.text[:300]) + assert r.headers["location"] == "/ui/account/sessions?m=revoke_missed" + assert await service.identity_for_token(theirs.token, activity=False) is not None, ( + "a missed revoke ended another account's session" + ) + rows = await engine.store.list_audit(action="auth.session_revoke_missed", limit=10) + assert len(rows) == 1, "one miss, one row" + assert rows[0]["actor"] == "a" + assert rows[0]["client"] == "127.0.0.1", "the miss row dropped the caller's address" + # The cut id and nothing else, so the row records no ownership either. + assert json.loads(str(rows[0]["detail"])) == {"session": target[:12]} diff --git a/tests/test_self_target_refusal_audit.py b/tests/test_self_target_refusal_audit.py index 8c41395e6b..02a9ba70a9 100644 --- a/tests/test_self_target_refusal_audit.py +++ b/tests/test_self_target_refusal_audit.py @@ -137,3 +137,64 @@ async def test_another_account_writes_no_refusal_row( r = await c.request(method, f"/users/{other.id}{suffix}", json=body, headers=_auth(tok)) assert r.status_code == 200, r.text assert await engine.store.list_audit(action="auth.self_target_refused", limit=10) == [] + + +# --- vault BACKLOG #3260, limb 2: a self-service revoke that ends nothing -------------------------- + +#: A well-formed session id that names no session. +_UNKNOWN_SESSION = "0" * 64 + + +async def test_a_missed_session_revoke_writes_one_row_and_the_same_answer(engine: Engine) -> None: + """RED when: ``DELETE /me/sessions/{id}`` answers its 404 with no audit row of its own (vault + BACKLOG #3260, limb 2; owner ruling 2026-10-16), or the answer or the row tells an unknown id + from another account's. + + One row per miss, with the id cut to 12 characters, and nothing else in its detail.""" + service = await _service(engine) + await _add(service, "a", Role.VIEWER) + await _add(service, "b", Role.VIEWER) + async with _client(engine, service) as c: + ta = (await _login(c, "a")).json()["token"] + tb = (await _login(c, "b")).json()["token"] + foreign = (await c.get("/me/sessions", headers=_auth(tb))).json()["sessions"][0]["id"] + answers = [] + # The grant is single-use, so each attempt takes its own re-proof, as a caller's would. + for target in (_UNKNOWN_SESSION, foreign): + _r, ta = await _reauth(c, ta, purpose="session_terminate") + assert _r.status_code == 200 + r = await c.delete(f"/me/sessions/{target}", headers=_auth(ta)) + answers.append((r.status_code, r.json())) + still = await c.get("/auth/me", headers=_auth(tb)) + assert answers == [(404, {"detail": "no such session"})] * 2, answers + assert still.status_code == 200, "a missed revoke ended another account's session" + rows = await engine.store.list_audit(action="auth.session_revoke_missed", limit=10) + assert len(rows) == 2, "one miss, one row" + # Newest first: the foreign id's row, then the unknown id's. + assert [json.loads(str(r["detail"])) for r in rows] == [ + {"session": foreign[:12]}, + {"session": _UNKNOWN_SESSION[:12]}, + ] + for row in rows: + assert row["actor"] == "a" + assert row["client"] == _DEFAULT_PEER[0], "the miss row dropped the caller's address" + assert await engine.store.list_audit(action="auth.session_revoked", limit=10) == [] + + +async def test_a_revoke_of_your_own_session_writes_no_miss_row(engine: Engine) -> None: + """The control: a revoke that ends a session writes ``auth.session_revoked``, not a miss.""" + service = await _service(engine) + await _add(service, "u", Role.VIEWER) + async with _client(engine, service) as c: + t1 = (await _login(c, "u")).json()["token"] + t2 = (await _login(c, "u")).json()["token"] + _r, t2 = await _reauth(c, t2, purpose="session_terminate") + sessions = (await c.get("/me/sessions", headers=_auth(t2))).json()["sessions"] + other = next(s for s in sessions if not s["current"]) + r = await c.delete(f"/me/sessions/{other['id']}", headers=_auth(t2)) + gone = await c.get("/auth/me", headers=_auth(t1)) + assert r.status_code == 200, r.text + assert gone.status_code == 401 + assert await engine.store.list_audit(action="auth.session_revoke_missed", limit=10) == [] + revoked = await engine.store.list_audit(action="auth.session_revoked", limit=10) + assert len(revoked) == 1 and revoked[0]["client"] == _DEFAULT_PEER[0] From c8fae10f3e1348df5aa979b71b96923168d76967 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 17:54:48 -0500 Subject: [PATCH 07/27] OIDC step-up wave 1 follow-ups from engine PR 2078 (BACKLOG #3145) Each item fixed or declined in one written line: 1. Declined. A gate-refused restart paging config_changed on every attempt is #2597's crash-loop contract, pinned by test_the_alert_fires_even_when_the_start_then_fails (comment in the app.py lifespan). 2. Declined the same way: the audit-chain walk runs inside engine.start(), and moving it ahead of the notice gate reorders that start (comment above auth.initialize()). 3. Built in part. build_idp_opener_from_settings in auth/oidc_http.py assembles the arguments for both call sites, AuthService.__init__ and verify/federation.py::_idp_opener; each keeps its own error mapping. The checks.py settings-preamble refactor is declined: it would collide with vault #3063 on the same helper. 4. Built. A step-up refusal made after the flow was redeemed carries start_client and client_moved through _step_up_address_fields, the helper the success row now uses too. They come from the staged flow, never the token; extra stays first. A refusal before the state check (or with no flow) carries neither. 5. Built. SECURITY.md documents the two fields; PHI.md's audit_log section row no longer says writers "only ever" store shapes, counts and ids, and the detail row names the client-address fields. 6. (a) Declined: the client raises IdpStepUpRequired before the registered handler because that handler can only re-prove a password, which POST /me/reauth refuses for this session. (b) Built as wording: the three IdP refusals say to repeat the action in the web console, which opens /ui/reauth for it; a bare /ui/reauth still bounces an unregistered next to /ui, unchanged. (c) Built: _step_up_refusal catches a store fault reading the mechanism (driver errors, RuntimeError, OSError, aiosqlite's ValueError; AUDIT_WRITE_DEFECTS re-raised) and keeps the 403 with the password wording instead of a 500. The one extra session read on a refusal is declined: it is the read that picks the wording. (d) Built: STEP_UP_VIA_HEADER / STEP_UP_VIA_IDP live in api/auth_models.py, which the FastAPI-free apiclient already imports. --- .../3145-oidc-step-up-follow-ups.security.md | 8 ++ docs/PHI.md | 4 +- docs/SECURITY.md | 8 +- messagefoundry/api/app.py | 9 +- messagefoundry/api/auth_models.py | 7 + messagefoundry/api/auth_routes.py | 16 ++- messagefoundry/api/security.py | 38 +++++- messagefoundry/apiclient/client.py | 17 ++- messagefoundry/auth/oidc_http.py | 28 +++- messagefoundry/auth/service.py | 128 ++++++++++++------ messagefoundry/verify/federation.py | 10 +- tests/test_auth_oidc_service.py | 3 +- tests/test_oidc_json_step_up_hint.py | 35 +++++ tests/test_oidc_step_up.py | 48 +++++++ 14 files changed, 291 insertions(+), 68 deletions(-) create mode 100644 changelog.d/3145-oidc-step-up-follow-ups.security.md diff --git a/changelog.d/3145-oidc-step-up-follow-ups.security.md b/changelog.d/3145-oidc-step-up-follow-ups.security.md new file mode 100644 index 0000000000..c40a970b03 --- /dev/null +++ b/changelog.d/3145-oidc-step-up-follow-ups.security.md @@ -0,0 +1,8 @@ +- **A refused federated step-up records where it started.** An `auth.reauth` row for a step-up + refused after its flow was redeemed now carries `start_client` and `client_moved`, as the row of + a completed step-up does. Both come from the flow the start leg staged, never from the token. A + refusal before the `state` check carries neither. The refusal a step-up gate sends an `oidc` + session now says to repeat the action in the web console, because a bare `/ui/reauth` names no + action and returns to `/ui`. A store fault while that refusal reads the session's sign-in + mechanism now keeps the `403`, with the password wording, rather than answering `500`. See + [SECURITY.md](../docs/SECURITY.md). (`vault BACKLOG #3145`) diff --git a/docs/PHI.md b/docs/PHI.md index 600fc3579f..d3db7efce9 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -194,7 +194,7 @@ destruction) are documented in [§3](#3-encryption-at-rest) under the matching h | `queue.handler_name` / `destination_name` / `channel_id` | all three | No — names, not bodies | No (metadata, deliberately not ciphered) | **PL-4** | The handler the transform worker runs; the destination the delivery worker drains | `n/a — not PHI` | | `messages.control_id`, `messages.message_type` | all three | Low (MSH-10/MSH-9) | **No** — plaintext by design | **PL-4** | Needed plaintext for dedup/routing/indexes (`ix_messages_control`). Covered only by the whole-DB / volume layer | `keep-forever by design` — dedup/routing keys that live and die with the message row | | `messages.origin`, `messages.origin_actor` (vault BACKLOG #2615) | all three | **No** — a fixed label and a username, never message content | **No** — plaintext by design | **PL-4** | How the row came to exist: `partner`, `operator_edit`, `operator_upload` or `reingress`, and for the two operator origins the acting user. Written at insert and never updated. Plain on purpose, because it must outlive the body and the ciphered `metadata`, which retention blanks. `NULL` on a row written before the column existed means not recorded. See [§6](#6-audit--accountability) | `keep-forever by design` — provenance that lives and dies with the message row | -| `audit_log.detail` | all three | Low — exposed IDs/counts, not bodies | **No** — plaintext by design | **PL-4** | JSON metadata about PHI *access* and other actions, not the PHI itself. Its writers store at least filter shapes, counts and ids. Some also store more: an edit-resend row holds **keyed** HMAC-SHA256 digests of the original and the edited body, and an upload-inject row one of the injected body. Without the store key a digest cannot be recovered or tested against a guess ([§6](#6-audit--accountability), vault BACKLOG #2615). And some store bounded engine-authored text: at least the `connection_test` and `connection_credential_test` rows carry the probe outcome, a fixed line or `safe_text()`/`safe_exc()`-scrubbed connector error text (BACKLOG #2372). None stores a message body. *Corrected 2026-10-07:* this said the writers "only ever store filter shapes, counts and ids" | `keep-forever by design` — 45 CFR 164.316(b)(2)(i) six-year documentation retention. **Not** chain-breakage: which rows a delete removes decides that, and the reasoning is stated once, in the `audit_days` row of [CONFIGURATION.md](CONFIGURATION.md#retention) | +| `audit_log.detail` | all three | Low — exposed IDs/counts, not bodies | **No** — plaintext by design | **PL-4** | JSON metadata about PHI *access* and other actions, not the PHI itself. Its writers store at least filter shapes, counts and ids. Some also store more: an edit-resend row holds **keyed** HMAC-SHA256 digests of the original and the edited body, and an upload-inject row one of the injected body. Without the store key a digest cannot be recovered or tested against a guess ([§6](#6-audit--accountability), vault BACKLOG #2615). And some store bounded engine-authored text: at least the `connection_test` and `connection_credential_test` rows carry the probe outcome, a fixed line or `safe_text()`/`safe_exc()`-scrubbed connector error text (BACKLOG #2372). Some store a client address beside the row's own `client`: at least the federated step-up's `auth.reauth` row, whose `start_client` is the address that started the step-up and whose `client_moved` says whether the callback came from another host (BACKLOG #2160, vault BACKLOG #3145). None stores a message body. *Corrected 2026-10-07:* this said the writers "only ever store filter shapes, counts and ids" | `keep-forever by design` — 45 CFR 164.316(b)(2)(i) six-year documentation retention. **Not** chain-breakage: which rows a delete removes decides that, and the reasoning is stated once, in the `audit_days` row of [CONFIGURATION.md](CONFIGURATION.md#retention) | | `audit_log.client` (ADR 0150) | all three | **No** — a network address; identifies a *host*, not a patient | No (metadata, deliberately not ciphered) | **PL-4** | The caller's client address — the "from where" of an audited action; `NULL` for engine-internal/`system` writes. **Personal data, but not PHI**, and exactly what HIPAA §164.312(b) audit controls exist to capture. Plaintext by decision: it must stay greppable/indexable for incident response, it already appears in the clear in `sessions.client`, and it is folded **inside** the tamper-evident hash chain — so it carries **integrity** protection even without confidentiality. Widens a store-file compromise from *who did what* to *who did what from where*; volume encryption + owner-only ACLs on whichever host owns the files — the engine's own `_secure_file` covers the **SQLite** store only ([§10](#10-secure-deployment--operations-checklist)) — are the control | `keep-forever by design` — same `audit_log` row lifetime as `detail`; the value is folded **inside** the hash chain | | `delivered_keys` (H2 idempotency ledger) | all three | **No** — hashes + ids only | No (deliberately not ciphered — nothing to protect) | **PL-4** | One row per completed outbound delivery: a SHA-256 `delivery_key` over non-PHI ids + a replay-stable seq, plus `outbox_id`/`message_id`/`destination_name`/`delivery_seq`. **Never a body or any PHI** — `control_id` is only *folded into the hash input*, never stored in the clear here. Lets the FIFO claim skip-and-complete a re-claimed already-delivered head without re-sending | `keep-forever by design` — the idempotency ledger a re-claimed already-delivered row checks instead of re-sending | | `state.namespace` / `state.key` | all three | **Possibly** — a Handler that keys correlation state on a raw MRN stores that identifier here in the clear | **No** — plaintext by construction: the pair is the composite primary key **and** the AAD input for `state.value`, so it cannot be ciphered without losing the lookup | **PL-4** | Authors must key state on a **surrogate, never a raw identifier**. Covered only by the whole-DB / volume layer. Rides `[retention].state_max_age_days` with its value | ``rides `[retention].state_max_age_days` `` | @@ -1805,7 +1805,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **2. `uvicorn` request/access log** (sub-stream of 1) | one line per HTTP request — method, **full request line including the query string**, status, timing | inherits stream 1's format | inherits stream 1's sink | request tracing, latency and error triage | inherits stream 1's | inherits stream 1's | **Can contain PHI.** `configure_logging` clears uvicorn's own handlers and propagates to the root, so the four filters apply; `serve` passes `log_config=None` and never disables `access_log`, so at the default `INFO` level every request is logged. OIDC `code`/`state` **are** scrubbed. **Not** scrubbed: a `?content=…` / `?field_value=…` that arrives anyway — no route has declared either since BACKLOG #1184, but the access line is built from the raw `query_string` rather than from what a route binds (the single-token residual above) | | **3. `messagefoundry.audit` off-box tee** (sub-stream of 1) | one JSON object per **committed** `audit_log` row: `event`/`ts`/`action`/`actor`/`channel_id`/`client`/`detail`, plus the `row_id`/`row_hash` anchor pair (BACKLOG #1198) that lets a collector tie the copy back to the row it came from and detect a gap in the chain | JSON | emitted after the row is durably committed and **outside** the store write lock; rides stream 1's handlers | shipping audit evidence to a SIEM so it survives a host compromise | inherits stream 1's | inherits stream 1's | `detail` is passed through the `safe_text` PHI chokepoint **before** it leaves the process; `client` is forwarded verbatim as a discrete field so a SIEM can index it. The `row_hash` anchor carries no PHI directly, but on a **keyless** store it is a plain SHA-256 whose other preimage members travel in the same record or the one before it, so a reader of the forwarded stream could test offline guesses at a span `safe_text` cut; it is a keyed MAC once the store cipher is active. Best-effort: a logging failure is caught, never raised into the audit write. **Pinned to `INFO`** — it is emitted even at `[logging].level = WARNING` | | **4. Off-box syslog/SIEM forwarder** — the shared **transport** for 1–3 | a copy of every record from 1–3 | `forward_format`, default **JSON** (independent of the stdout format) | the operator's collector (`forward_host`/`_port`) | off-box evidence retention / SIEM correlation | **default-on when a collector is named.** Transport: `udp` (default) / `tcp` / **`tls`** (RFC 5425, CA-anchored, verified by default). `serve` gates the hop on the shared posture gradient before the handler is installed: verified TLS ungated; otherwise loopback / attested ALLOW, non-enforcing WARN, **enforcing REFUSE (exit 2)** — there is no synthetic arm, and [ADR 0186](adr/0186-retire-the-synthetic-data-declaration-every-instance-carries-patient-data.md) left this cell with no per-hop way to accept the risk at all | the collector's, not the engine's | the identical four filters are installed on this handler. So the forwarded copy gets the same best-effort PHI redaction and credential scrub as stdout. A single-token identifier can survive it, as it can there. The copy also carries usernames, connection names, message ids, client addresses and the audit chain. That is the engine's own stated reason for gating the hop | -| **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers only ever store filter shapes, counts and ids — never bodies or credentials — plus the store ACL and the volume layer | +| **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | | **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 -- or a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 880d7999c9..3d4c48e79c 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -3855,7 +3855,13 @@ token claim**. There is no new `auth_provider` value: a federated login resolves `/ui/login?e=rate_limited` and a WARNING log), a return the `Sec-Fetch-Mode` check refuses (audited `auth.login_failed`), and any refusal at the start leg `POST /ui/reauth/oidc`. A refusal those two methods decide changes nothing and re-renders the step-up page with a plain message, except one that finds the session gone, which sends the operator to sign in. A refusal after the `state` - check is filed under the staged session's account. A cancel at the IdP returns to the same page. + check is filed under the staged session's account. That row, and the row of a completed step-up, + also carry `start_client`, the address the start leg staged on the flow, and `client_moved`, + whether the callback came from another host by the new-address comparison, or null when either + address is unknown (BACKLOG #2160, vault BACKLOG #3145). Neither is ever read from the token. A + refusal before the `state` check names neither. A move is recorded, never refused. Behind a proxy + the engine does not trust, both legs carry the proxy's address, so `client_moved` reads false + there. A cancel at the IdP returns to the same page. A callback without the flow's own `state` consumes nothing, and the flow waits for the real return. The sign-in callback follows the same rule, an `error=` or malformed one included (BACKLOG #3198). A password is diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 6370ef36ed..d406f33509 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -10276,7 +10276,9 @@ async def _audit_convergence_reload(outcome: ReloadOutcome) -> None: # This process cannot vouch for a config it never checked: its flag rows say so. engine.config_baseline_unchecked = start_outcome in _UNCHECKED_OUTCOMES # Raised here, before engine.start(), so new bytes that make the start itself fail - # still page on every attempt of a crash loop. Never raises. + # still page on every attempt of a crash loop. Never raises. Vault BACKLOG #3145 + # item 1 (a gate-refused restart pages on every attempt) is declined: that is this + # contract, pinned by test_the_alert_fires_even_when_the_start_then_fails (#2597). if start_config is not None and start_config.changed: _raise_config_changed(notifier or LoggingAlertSink(), engine, start_config) except BaseException: @@ -10367,7 +10369,10 @@ async def _audit_convergence_reload(outcome: ReloadOutcome) -> None: # any connection starts rather than after. Both touch only the store: initialize() # seeds the built-in roles and the gate reads the users. A refusal here also comes # before the start's opt-in audit-chain walk, so that walk cannot alert on such a - # start. Everything else that uses the service stays below the start. + # start. Vault BACKLOG #3145 item 2 is declined: the walk runs inside engine.start(), + # and moving it ahead of this gate reorders the start whose failure paging + # test_the_alert_fires_even_when_the_start_then_fails pins (#2597). Everything else + # that uses the service stays below the start. await auth.initialize() app.state.auth = auth await _assert_security_notice_is_deliverable( diff --git a/messagefoundry/api/auth_models.py b/messagefoundry/api/auth_models.py index 8dce0ff434..04e6b8f41d 100644 --- a/messagefoundry/api/auth_models.py +++ b/messagefoundry/api/auth_models.py @@ -34,6 +34,13 @@ # A session token is 43 URL-safe characters (auth/tokens.py); the cap only refuses absurd input. _TOKEN_MAX = 256 +#: The response header an ``oidc`` session's step-up refusal carries, set to +#: :data:`STEP_UP_VIA_IDP`, so a client can branch on a header rather than parse the detail +#: (BACKLOG #2158). Here, with the auth wire models, so the engine's raise sites and the +#: FastAPI-free ``apiclient`` share one spelling (vault BACKLOG #3145 item 6). +STEP_UP_VIA_HEADER = "X-Step-Up-Via" +STEP_UP_VIA_IDP = "idp" + class CredentialReply(BaseModel): """A response model whose body carries a live credential: a session token, a staged TOTP seed, diff --git a/messagefoundry/api/auth_routes.py b/messagefoundry/api/auth_routes.py index 8f2348bcc7..73a94cb303 100644 --- a/messagefoundry/api/auth_routes.py +++ b/messagefoundry/api/auth_routes.py @@ -29,6 +29,8 @@ from messagefoundry.api._ui_seam import AdminHandlers from messagefoundry.api.auth_models import ( _GROUP_MAX, + STEP_UP_VIA_HEADER, + STEP_UP_VIA_IDP, AdGroupMap, AdGroupMapEntry, AdGroupScopeEntry, @@ -749,13 +751,14 @@ async def reauth( # the message names the leg that works. The header is the one the step-up gates # send such a session (BACKLOG #2158), so a client branches on it here too rather # than parsing the detail. No X-Step-Up-Required: this is the step-up itself. - # Typed out because api/security.py keeps its constant private; a test pins the - # name and the value to what the gates send. + # The constants the gates send too (vault BACKLOG #3145); a test pins the two + # to each other. raise HTTPException( status.HTTP_403_FORBIDDEN, "this session was signed in through the identity provider; re-authenticate" - " there through the web console at /ui/reauth, not with a password", - headers={"X-Step-Up-Via": "idp"}, + " there, not with a password: repeat the action in the web console, which" + " opens /ui/reauth for it", + headers={STEP_UP_VIA_HEADER: STEP_UP_VIA_IDP}, ) if elevation.directory_unconfirmed: # BACKLOG #2027: the directory could not judge the password, so "failed" would @@ -802,8 +805,9 @@ async def mfa_verify( raise HTTPException( status.HTTP_403_FORBIDDEN, "this session was signed in through the identity provider and owes no code;" - " step up there through the web console at /ui/reauth", - headers={"X-Step-Up-Via": "idp"}, + " step up there by repeating the action in the web console, which opens" + " /ui/reauth for it", + headers={STEP_UP_VIA_HEADER: STEP_UP_VIA_IDP}, ) if elevation.directory_unconfirmed: # BACKLOG #2023: the code was never checked, and the token still authenticates, so a diff --git a/messagefoundry/api/security.py b/messagefoundry/api/security.py index 29e325f32d..5315f5a4b1 100644 --- a/messagefoundry/api/security.py +++ b/messagefoundry/api/security.py @@ -18,6 +18,7 @@ import time from collections.abc import Awaitable, Callable, Coroutine, Mapping from dataclasses import dataclass +from functools import cache from typing import Any, get_args from fastapi import ( @@ -34,11 +35,12 @@ from pydantic import BaseModel from starlette.requests import HTTPConnection -from messagefoundry.api.auth_models import CredentialReply +from messagefoundry.api.auth_models import STEP_UP_VIA_HEADER, STEP_UP_VIA_IDP, CredentialReply from messagefoundry.api.tls_client_cert import ( MF_CLIENT_PEERCERT_STATE_KEY, peercert_from_ssl_object, ) +from messagefoundry.audit_write import AUDIT_WRITE_DEFECTS from messagefoundry.auth import AuthProvider, Identity, Permission, Role from messagefoundry.auth.notifications import deadline_utc as deadline_utc # re-export from messagefoundry.auth.service import AuthService, LoginOutcome @@ -52,6 +54,7 @@ from messagefoundry.pipeline.alerts import AlertSink, LoggingAlertSink from messagefoundry.pipeline.cert_expiry import peer_cert_expiry from messagefoundry.redaction import safe_exc +from messagefoundry.store.base import store_driver_errors from messagefoundry.store.store import UserRecord log = logging.getLogger(__name__) @@ -1123,11 +1126,22 @@ async def dependency(request: Request) -> Identity: #: session (BACKLOG #2142, ADR 0142 Amendment F). _IDP_STEP_UP_DETAIL = ( "step-up re-verification required; this session steps up at the identity provider, in a" - " browser, through the web console at /ui/reauth, not with a password" + " browser, not with a password: repeat the action in the web console, which opens /ui/reauth" + " for it" ) #: Set to ``idp`` on an ``oidc`` session's step-up refusal, so a client can branch on a header #: rather than parse the detail. Absent on every other refusal. -_STEP_UP_VIA_HEADER = "X-Step-Up-Via" +_STEP_UP_VIA_HEADER = STEP_UP_VIA_HEADER + + +@cache +def _session_read_faults() -> tuple[type[Exception], ...]: + """What a refused session read raises: the drivers' errors, plus ``RuntimeError`` and + ``OSError`` as :func:`~messagefoundry.store.base.store_driver_errors` asks, and aiosqlite's + ``ValueError`` for a closed connection. A caller re-raises + :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` first, so a bug is not read as a fault. + Built on first use, because naming a server driver imports it.""" + return (RuntimeError, OSError, ValueError, *store_driver_errors()) async def _step_up_refusal( @@ -1142,8 +1156,22 @@ async def _step_up_refusal( headers = {"X-Step-Up-Required": "1"} if action is not None: headers["X-Step-Up-Action"] = action - if await auth.session_steps_up_at_idp(token): - headers[_STEP_UP_VIA_HEADER] = "idp" + try: + via_idp = await auth.session_steps_up_at_idp(token) + except AUDIT_WRITE_DEFECTS: + raise + except _session_read_faults() as exc: + # Vault BACKLOG #3145 item 6: the read only picks the wording, and the gate has already + # refused, so a store fault keeps the 403 rather than becoming a 500. It answers with the + # password wording; an oidc client's POST /me/reauth then asks again and is told the IdP. + log.warning( + "a step-up refusal could not read the session's sign-in mechanism (%s); it answers" + " with the password wording", + type(exc).__name__, + ) + via_idp = False + if via_idp: + headers[_STEP_UP_VIA_HEADER] = STEP_UP_VIA_IDP return HTTPException(status.HTTP_403_FORBIDDEN, _IDP_STEP_UP_DETAIL, headers=headers) return HTTPException(status.HTTP_403_FORBIDDEN, _STEP_UP_DETAIL, headers=headers) diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index ee4338daa1..c5eeeff6a9 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -29,6 +29,8 @@ from pydantic import BaseModel, ValidationError from messagefoundry.api.auth_models import ( + STEP_UP_VIA_HEADER, + STEP_UP_VIA_IDP, AdGroupMap, AdGroupMapEntry, AuditList, @@ -147,8 +149,10 @@ class IdpStepUpRequired(ApiError): Raised on a 403 carrying ``X-Step-Up-Via: idp``, which the engine sends for a session minted by the federated (``oidc``) sign-in. ``POST /me/reauth`` refuses such a session, so this client - raises instead of prompting for a password. Its step-up runs in the web console at - ``/ui/reauth``, through the IdP, and this client cannot perform it: that leg needs a browser and + raises instead of prompting for a password. Its step-up runs in the web console, through the + IdP: repeating the action there sends the browser to ``/ui/reauth`` with that action named + (a bare ``/ui/reauth`` names none and returns to ``/ui``; vault BACKLOG #3145). This client + cannot perform it: that leg needs a browser and re-keys the session it steps up. A subclass of :class:`ApiError`, so a caller that catches only that still sees the refusal.""" @@ -1034,8 +1038,13 @@ def _request( **kw, ) # BACKLOG #2158: an ``oidc`` session steps up at the IdP, in a browser. POST /me/reauth - # refuses it, so the password prompt below would only earn a second 403. - if response.status_code == 403 and response.headers.get("X-Step-Up-Via") == "idp": + # refuses it, so the password prompt below would only earn a second 403. Vault BACKLOG + # #3145 item 6 asked to run the registered handler first; declined, because that handler + # can only re-prove a password, which is the second 403. + if ( + response.status_code == 403 + and response.headers.get(STEP_UP_VIA_HEADER) == STEP_UP_VIA_IDP + ): raise IdpStepUpRequired(_error_detail(response), status=response.status_code) # Step-up re-verification (ASVS 7.5.3): the engine refuses a sensitive op with 403 + # X-Step-Up-Required when this session hasn't re-proved its credential recently. Prompt the diff --git a/messagefoundry/auth/oidc_http.py b/messagefoundry/auth/oidc_http.py index 6bfd7ef22a..3c5fe80a21 100644 --- a/messagefoundry/auth/oidc_http.py +++ b/messagefoundry/auth/oidc_http.py @@ -42,6 +42,7 @@ import ssl import urllib.request from collections.abc import Callable +from typing import TYPE_CHECKING from messagefoundry.auth.oidc.jwks import _MAX_JWKS_BYTES from messagefoundry.auth.trust_anchors import oidc_anchor_spec, verified_anchor_cadata @@ -59,7 +60,10 @@ reply_framing_fault, ) -__all__ = ["build_idp_opener", "jwks_fetcher"] +if TYPE_CHECKING: + from messagefoundry.config.settings import AuthSettings + +__all__ = ["build_idp_opener", "build_idp_opener_from_settings", "jwks_fetcher"] #: Wall-clock bound for a single IdP HTTP round trip. Deliberately short: both legs run inside a #: browser request, and a hung IdP must degrade the federated login, never pin a worker thread. @@ -143,6 +147,28 @@ def build_idp_opener( return build_strict_opener(_NoRedirectHandler, urllib.request.HTTPSHandler(context=ctx)) +def build_idp_opener_from_settings( + settings: AuthSettings, *, enforcing: bool, anchor: bool = True +) -> urllib.request.OpenerDirector: + """:func:`build_idp_opener` with its arguments read from ``[auth]``, so the engine and the + federation check cannot read the anchor, pin or CRL file differently (vault BACKLOG #3145 item + 3). Only the argument assembly is shared: each caller keeps its own answer to a failure, a + refused start in ``AuthService`` and a check row in ``verify.federation``. + + ``anchor=False`` leaves the anchor and its pin out, for the federation check's revocation row + alone. The CRL file is the hop's own knob (BACKLOG #299) and is passed either way. + + The settings preamble the check legs in ``messagefoundry/checks.py`` repeat is not shared here: + that refactor is declined, because it would collide with vault BACKLOG #3063 on the same + helper.""" + return build_idp_opener( + settings.oidc_tls_ca_cert_file if anchor else None, + pin=settings.oidc_tls_ca_cert_pin if anchor else None, + enforcing=enforcing, + crl_file=settings.oidc_tls_crl_file, + ) + + def jwks_fetcher( jwks_uri: str, opener: urllib.request.OpenerDirector, diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 4335c5e8aa..95faa6c05a 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -91,7 +91,7 @@ notice_kind_log_label, ) from messagefoundry.auth.oidc import PendingFlow -from messagefoundry.auth.oidc_http import build_idp_opener, jwks_fetcher +from messagefoundry.auth.oidc_http import build_idp_opener_from_settings, jwks_fetcher from messagefoundry.auth.passwords import hash_password, needs_rehash, verify_password from messagefoundry.auth.permissions import ( CUSTOM_ROLE_ID_PREFIX, @@ -2622,13 +2622,10 @@ def __init__( # preflight honors [security].enforcement (warn ≠ refuse) — a pin mismatch always refuses, # a group/world-writable DACL refuses only at enforce. Without the dial this seam would # hardcode enforce and make warn-mode startup unreachable for the OIDC anchor. - self._oidc_opener = build_idp_opener( - settings.oidc_tls_ca_cert_file, - pin=settings.oidc_tls_ca_cert_pin, - enforcing=self._trust_anchors_enforcing, - # BACKLOG #299: revocation checking against the IdP certificate. This hop resolves no - # trust anchor, so it carries its own CRL setting rather than inheriting [tls].crl_file. - crl_file=settings.oidc_tls_crl_file, + # BACKLOG #299: the CRL file is this hop's own knob, not [tls].crl_file; the helper + # passes it. The federation check builds through the same helper (vault #3145). + self._oidc_opener = build_idp_opener_from_settings( + settings, enforcing=self._trust_anchors_enforcing ) # BACKLOG #1887: must follow the opener, whose finished context it reads. _refuse_idp_revocation(settings, self._oidc_opener, hop_posture) @@ -4848,9 +4845,11 @@ async def abandon_oidc_step_up( return flow actor = await self._step_up_actor(flow) if flow.step_up_session_hash is None: - return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) + return await self._step_up_refused( + FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow + ) return await self._step_up_refused( - reason, actor=actor, client=client, return_to=flow.return_to + reason, actor=actor, client=client, return_to=flow.return_to, flow=flow ) async def _redeem_step_up_flow( @@ -4910,8 +4909,9 @@ async def complete_oidc_step_up( The session is re-anchored to the callback's address when the callback has one. The ``auth.reauth`` row a completed step-up writes, including one whose rotation lost the session, also records the start leg's address as ``start_client`` and whether the two are - different hosts as ``client_moved`` (BACKLOG #2160). The refusal rows above carry neither. - A move is recorded, never refused. Behind a proxy the engine does not trust, both legs carry + different hosts as ``client_moved`` (BACKLOG #2160). So does a refusal made after the flow + was redeemed (vault BACKLOG #3145); one made before carries neither. A move is recorded, + never refused. Behind a proxy the engine does not trust, both legs carry the proxy's address, so ``client_moved`` reads false there. Not padded to a deadline, unlike the sign-in callback: the caller already holds a session, @@ -4926,7 +4926,9 @@ async def complete_oidc_step_up( actor = await self._step_up_actor(flow) token_hash = flow.step_up_session_hash if token_hash is None: - return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) + return await self._step_up_refused( + FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow + ) # BACKLOG #2301 (ASVS 2.4.2): max_age=0 and prompt=login make the person authenticate at the # IdP, so a step-up callback faster than a person can do that is refused. Before the code is # redeemed, so the refused flow spends nothing at the token endpoint. @@ -4936,7 +4938,7 @@ async def complete_oidc_step_up( arrived_too_early = self._oidc_callback_too_early(flow) if arrived_too_early and not self._oidc_callback_floor_exempt_values(): return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to + TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow ) try: principal_claims = await asyncio.to_thread( @@ -4945,7 +4947,7 @@ async def complete_oidc_step_up( except oidc.ClaimsError as exc: # exc.reason is closed-set, so nothing IdP-influenced reaches the audit row. return await self._step_up_refused( - exc.reason, actor=actor, client=client, return_to=return_to + exc.reason, actor=actor, client=client, return_to=return_to, flow=flow ) except oidc.TokenRefusedError as exc: # The endpoint answered. As on the sign-in leg, that is not an outage (#1948), and the @@ -4956,6 +4958,7 @@ async def complete_oidc_step_up( client=client, return_to=return_to, extra={"status": exc.status}, + flow=flow, ) except (OSError, ValueError, http.client.HTTPException) as exc: # The sign-in leg's outage arm, for the same reasons (#1948). Only the exception TYPE, @@ -4967,6 +4970,7 @@ async def complete_oidc_step_up( client=client, return_to=return_to, extra=_idp_error_fields(exc), + flow=flow, ) self.mark_oidc_unavailable(type(exc).__name__) return await self._step_up_refused( @@ -4975,6 +4979,7 @@ async def complete_oidc_step_up( client=client, return_to=return_to, extra=_idp_error_fields(exc), + flow=flow, ) floor_exempt_amr = ( self._oidc_callback_floor_exemption(principal_claims.amr) if arrived_too_early else () @@ -4982,7 +4987,7 @@ async def complete_oidc_step_up( if arrived_too_early and not floor_exempt_amr: # The opted-in site's too-early callback whose amr names no exempt value (BACKLOG #2388). return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to + TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow ) now = time.time() session = await self._load_session(token_hash) @@ -4996,12 +5001,22 @@ async def complete_oidc_step_up( or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): return await self._step_up_refused( - "session_gone", actor=actor, client=client, return_to=return_to, lost=True + "session_gone", + actor=actor, + client=client, + return_to=return_to, + lost=True, + flow=flow, ) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: return await self._step_up_refused( - "session_gone", actor=actor, client=client, return_to=return_to, lost=True + "session_gone", + actor=actor, + client=client, + return_to=return_to, + lost=True, + flow=flow, ) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, @@ -5016,7 +5031,7 @@ async def complete_oidc_step_up( # the claims ladder is filed as a subject mismatch whatever its auth_time. Both tests # below judge this session's identity, so neither is the right name for someone else. return await self._step_up_refused( - STEP_UP_SUBJECT_MISMATCH, actor=actor, client=client, return_to=return_to + STEP_UP_SUBJECT_MISMATCH, actor=actor, client=client, return_to=return_to, flow=flow ) # FRESHNESS, two tests, and both must pass. max_age=0 and prompt=login ask the IdP to # authenticate the user afresh, so a conforming IdP's auth_time postdates this request. @@ -5025,7 +5040,7 @@ async def complete_oidc_step_up( skew = self._settings.oidc_clock_skew_seconds if flow.issued_at <= 0 or principal_claims.auth_time < flow.issued_at - skew: return await self._step_up_refused( - STEP_UP_NOT_FRESH, actor=actor, client=client, return_to=return_to + STEP_UP_NOT_FRESH, actor=actor, client=client, return_to=return_to, flow=flow ) # (b) Against the IdP's own clock (BACKLOG #2143): auth_time must be LATER than the one the # session holds, which is the sign-in's or the last step-up's. No skew applies, because @@ -5041,11 +5056,19 @@ async def complete_oidc_step_up( held_auth_time = session.idp_auth_time if held_auth_time is None: return await self._step_up_refused( - STEP_UP_IDP_AUTH_TIME_MISSING, actor=actor, client=client, return_to=return_to + STEP_UP_IDP_AUTH_TIME_MISSING, + actor=actor, + client=client, + return_to=return_to, + flow=flow, ) if principal_claims.auth_time <= held_auth_time: return await self._step_up_refused( - STEP_UP_IDP_AUTH_TIME_NOT_LATER, actor=actor, client=client, return_to=return_to + STEP_UP_IDP_AUTH_TIME_NOT_LATER, + actor=actor, + client=client, + return_to=return_to, + flow=flow, ) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP @@ -5053,13 +5076,17 @@ async def complete_oidc_step_up( # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: return await self._step_up_refused( - "not_configured", actor=actor, client=client, return_to=return_to + "not_configured", actor=actor, client=client, return_to=return_to, flow=flow ) if not user.directory_object_id: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). return await self._step_up_refused( - DIRECTORY_OBJECT_ID_MISSING, actor=actor, client=client, return_to=return_to + DIRECTORY_OBJECT_ID_MISSING, + actor=actor, + client=client, + return_to=return_to, + flow=flow, ) try: principal = await asyncio.to_thread( @@ -5070,18 +5097,18 @@ async def complete_oidc_step_up( "directory lookup failed during a federated step-up: %s", type(exc).__name__ ) return await self._step_up_refused( - "directory_unavailable", actor=actor, client=client, return_to=return_to + "directory_unavailable", actor=actor, client=client, return_to=return_to, flow=flow ) if principal is None: return await self._step_up_refused( - "not_in_directory", actor=actor, client=client, return_to=return_to + "not_in_directory", actor=actor, client=client, return_to=return_to, flow=flow ) # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. mismatch = _directory_answer_mismatch(principal, user.directory_object_id) if mismatch is not None: return await self._step_up_refused( - mismatch, actor=actor, client=client, return_to=return_to + mismatch, actor=actor, client=client, return_to=return_to, flow=flow ) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups # map to, the same test _directory_step_up_refusal applies on the TOTP and passkey legs @@ -5092,11 +5119,16 @@ async def complete_oidc_step_up( fresh = await self._load_user(user.id) if fresh is None: return await self._step_up_refused( - "session_gone", actor=actor, client=client, return_to=return_to, lost=True + "session_gone", + actor=actor, + client=client, + return_to=return_to, + lost=True, + flow=flow, ) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): return await self._step_up_refused( - DIRECTORY_ROLES_DEMOTED, actor=actor, client=client, return_to=return_to + DIRECTORY_ROLES_DEMOTED, actor=actor, client=client, return_to=return_to, flow=flow ) purpose = flow.step_up_purpose # The password leg's three ORDER-CRITICAL steps (see :meth:`reauth`), against the hash. @@ -5123,14 +5155,9 @@ async def complete_oidc_step_up( self._grant_action_step_up(hash_token(elevation.token), purpose) self.clear_oidc_unavailable() # BACKLOG #2160: the start leg staged its caller's address, and the session is re-anchored - # to the callback's above. Record whether the address moved mid-ceremony, compared as the - # new-address signal compares. Recorded only: refusing a move, or anchoring to the start - # address, would change ADR 0142 Amendment B and needs a ruling first. None when either - # address is unknown, since an empty string matches nothing and proves no move. - start_client = flow.client_ip or None - client_moved = ( - not self._same_host(start_client, client) if start_client and client else None - ) + # to the callback's above. Record whether the address moved mid-ceremony. Recorded only: + # refusing a move, or anchoring to the start address, would change ADR 0142 Amendment B + # and needs a ruling first. detail: dict[str, object] = { "ok": elevation.ok, "provider": AuthProvider.AD.value, @@ -5139,8 +5166,7 @@ async def complete_oidc_step_up( "session_lost": elevation.session_lost, "grant_refused": grant_refused, "session_revoked": False, - "start_client": start_client, - "client_moved": client_moved, + **self._step_up_address_fields(flow, client), } if floor_exempt_amr: # BACKLOG #2388: present only when the exemption let a too-early callback through, and @@ -5160,6 +5186,21 @@ async def _step_up_actor(self, flow: PendingFlow) -> str: user = await self._load_user(session.user_id) if session is not None else None return user.username if user is not None else "" + def _step_up_address_fields(self, flow: PendingFlow, client: str | None) -> dict[str, object]: + """``start_client`` and ``client_moved`` for a step-up row (BACKLOG #2160, vault #3145). + + ``start_client`` is the address the start leg staged on the flow, never a value from the + IdP's token. ``client_moved`` compares it with the callback's as the new-address signal + compares (:meth:`_same_host`), and is None when either address is unknown, since an empty + string matches nothing and proves no move.""" + start_client = flow.client_ip or None + return { + "start_client": start_client, + "client_moved": ( + not self._same_host(start_client, client) if start_client and client else None + ), + } + async def _step_up_refused( self, reason: str, @@ -5170,11 +5211,17 @@ async def _step_up_refused( lost: bool = False, extra: Mapping[str, object] | None = None, flow_consumed: bool = True, + flow: PendingFlow | None = None, ) -> OidcStepUp: """Audit and package one refusal of the federated step-up leg. ``reason`` is closed-set. ``extra`` adds fields to the audit row only, such as a received status or an exception - type (BACKLOG #1948). It never carries IdP text.""" + type (BACKLOG #1948). It never carries IdP text. + + ``flow`` adds ``start_client`` and ``client_moved``, as the success row carries them (vault + BACKLOG #3145). Only a refusal made after the flow was redeemed passes it: before ``state`` + matched, the callback is not shown to belong to the flow, and some refusals have no flow + at all, so those rows carry neither field.""" # `extra` first, so it can never overwrite a closed-set field. detail: dict[str, object] = { **(extra or {}), @@ -5182,6 +5229,7 @@ async def _step_up_refused( "provider": AuthProvider.AD.value, "mech": "oidc", "reason": reason, + **(self._step_up_address_fields(flow, client) if flow is not None else {}), } await self._audit("auth.reauth", actor=actor, detail=_json(detail), client=client) return OidcStepUp( diff --git a/messagefoundry/verify/federation.py b/messagefoundry/verify/federation.py index bd2e1d7ba1..6ee2094a01 100644 --- a/messagefoundry/verify/federation.py +++ b/messagefoundry/verify/federation.py @@ -280,17 +280,15 @@ def _idp_opener( return the ``fed.idp_tls`` FAIL or ERROR row that says why it did not build. Reads the anchor and CRL files; opens no socket. ``anchor=False`` leaves the anchor and its pin out, for :func:`idp_revocation_result` alone.""" - from messagefoundry.auth.oidc_http import build_idp_opener + from messagefoundry.auth.oidc_http import build_idp_opener_from_settings from messagefoundry.auth.trust_anchors import TrustAnchorError from messagefoundry.config.settings import SecurityEnforcement - auth = settings.auth try: - return build_idp_opener( - auth.oidc_tls_ca_cert_file if anchor else None, - pin=auth.oidc_tls_ca_cert_pin if anchor else None, + return build_idp_opener_from_settings( + settings.auth, enforcing=settings.security.enforcement is SecurityEnforcement.ENFORCE, - crl_file=auth.oidc_tls_crl_file, + anchor=anchor, ) except OSError as exc: return CheckResult(*_TLS_ROW, Status.FAIL, f"could not build the pinned TLS context: {exc}") diff --git a/tests/test_auth_oidc_service.py b/tests/test_auth_oidc_service.py index 1d74d94213..035cd29975 100644 --- a/tests/test_auth_oidc_service.py +++ b/tests/test_auth_oidc_service.py @@ -348,7 +348,8 @@ def spy( ) return urllib.request.build_opener() - monkeypatch.setattr("messagefoundry.auth.service.build_idp_opener", spy) + # The helper both build sites share (vault BACKLOG #3145) calls the module-level builder. + monkeypatch.setattr("messagefoundry.auth.oidc_http.build_idp_opener", spy) # The CRL path must name a real file: [auth].oidc_tls_crl_file refuses a missing one at load # (BACKLOG #1997). The spy never reads it, so its content does not matter. crl = tmp_path / "idp-crl.pem" diff --git a/tests/test_oidc_json_step_up_hint.py b/tests/test_oidc_json_step_up_hint.py index 5c65fbd963..248287cae4 100644 --- a/tests/test_oidc_json_step_up_hint.py +++ b/tests/test_oidc_json_step_up_hint.py @@ -172,6 +172,9 @@ def _assert_idp_hint(site: str, r: httpx.Response, action: str | None) -> None: assert r.headers.get("X-Step-Up-Action") == action, site detail = r.json()["detail"] assert "/ui/reauth" in detail, (site, detail) + # Vault BACKLOG #3145 item 6: a bare /ui/reauth names no action and bounces to /ui, so the + # hint sends the reader back through the action, which reaches /ui/reauth with it named. + assert "repeat the action in the web console" in detail, (site, detail) assert "/me/reauth" not in detail, (site, detail) # BACKLOG #2142: a code opens no window on an oidc session, so the hint must not name it. assert "mfa-verify" not in detail, (site, detail) @@ -402,3 +405,35 @@ def _prompt() -> bool: client.close() assert not isinstance(raised.value, IdpStepUpRequired) assert prompted == ["step-up"] + + +def test_the_step_up_header_is_one_constant_on_both_sides() -> None: + """Vault BACKLOG #3145 item 6: the gates, the two route refusals and the FastAPI-free client + read one spelling, from the auth wire models, rather than four copies of a literal.""" + from messagefoundry.api import auth_models + from messagefoundry.apiclient import client as apiclient + + assert _STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" + assert apiclient.STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER + assert apiclient.STEP_UP_VIA_IDP is auth_models.STEP_UP_VIA_IDP == "idp" + package = Path(auth_models.__file__).parent.parent + for module in ("api/auth_routes.py", "apiclient/client.py", "api/security.py"): + text = (package / module).read_text(encoding="utf-8") + assert '"X-Step-Up-Via"' not in text, f"{module} spells the header out again" + + +async def test_a_store_fault_reading_the_mechanism_keeps_the_403( + engine: Engine, service: AuthService, monkeypatch: pytest.MonkeyPatch +) -> None: + """RED when: a store fault in the read that picks the refusal's wording turns the gate's 403 + into a 500 (vault BACKLOG #3145 item 6). It keeps the refusal and the password wording.""" + uid = await _add_admin(service) + token = await _session(service, uid, "tok-fault-oidc", mechanism="oidc") + + async def refused_read(token: str | None) -> bool: + raise OSError("the store connection dropped") + + monkeypatch.setattr(service, "session_steps_up_at_idp", refused_read) + async with _client_at(engine, service, HOME) as c: + r = await c.post("/me/mfa/enroll", headers=_auth(token)) + _assert_password_refusal("require_reauth_only_action", r, STEP_UP_ACTION_MFA_ENROLL) diff --git a/tests/test_oidc_step_up.py b/tests/test_oidc_step_up.py index fb47e46204..d953bb97ef 100644 --- a/tests/test_oidc_step_up.py +++ b/tests/test_oidc_step_up.py @@ -413,6 +413,51 @@ async def test_the_step_up_row_records_whether_the_address_moved_mid_ceremony( await store.close() +async def test_a_refusal_after_the_flow_is_redeemed_records_the_address_fields( + rsa_key: rsa.RSAPrivateKey, monkeypatch: pytest.MonkeyPatch +) -> None: + """RED when: a step-up refused after its flow was redeemed leaves out ``start_client`` and + ``client_moved``, which the success row carries (vault BACKLOG #3145 item 4). They come from the + flow the start leg staged, never from the token.""" + store = await MessageStore.open(":memory:") + try: + service = await _service(store, rsa_key) + token = await _oidc_session(service, monkeypatch, rsa_key) + flow_id, _url = await _begin(service, token, client="10.0.0.8") + stale = _staged(service, flow_id).issued_at - 2 * service._settings.oidc_clock_skew_seconds + out = await _return_from_idp( + service, monkeypatch, rsa_key, flow_id, client="10.0.0.9", auth_time=stale - 1 + ) + + assert not out.ok and out.reason == STEP_UP_NOT_FRESH + detail = json.loads(str((await _audit_rows(store, "auth.reauth"))[-1]["detail"])) + assert detail["reason"] == STEP_UP_NOT_FRESH + assert (detail["start_client"], detail["client_moved"]) == ("10.0.0.8", True) + finally: + await store.close() + + +async def test_a_refusal_before_the_state_matches_records_no_address_fields( + rsa_key: rsa.RSAPrivateKey, monkeypatch: pytest.MonkeyPatch +) -> None: + """The control for vault BACKLOG #3145 item 4: before ``state`` matches, the callback is not + shown to belong to the flow, so its row compares nothing with the flow's start address.""" + store = await MessageStore.open(":memory:") + try: + service = await _service(store, rsa_key) + token = await _oidc_session(service, monkeypatch, rsa_key) + flow_id, _url = await _begin(service, token, client="10.0.0.8") + out = await service.complete_oidc_step_up( + flow_id=flow_id, state="forged", code=AUTH_CODE, client="10.0.0.9", public_origin=ORIGIN + ) + + assert not out.ok and out.reason == "state_mismatch" + detail = json.loads(str((await _audit_rows(store, "auth.reauth"))[-1]["detail"])) + assert "start_client" not in detail and "client_moved" not in detail + finally: + await store.close() + + # --- the callback: every refusal elevates nothing ----------------------------------------------------- @@ -815,6 +860,9 @@ async def test_an_account_demoted_in_the_directory_since_sign_in_is_refused( "provider": "ad", "mech": "oidc", "reason": DIRECTORY_ROLES_DEMOTED, + # Both legs at the helpers' default address (vault BACKLOG #3145 item 4). + "start_client": "10.0.0.9", + "client_moved": False, } await _assert_untouched(service, token) assert set(await store.get_user_role_ids(user_id)) == {"operator"}, "roles were written" From 7a6961afd0c8b32c04ed41e9d1160b696d20a803 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 18:12:57 -0500 Subject: [PATCH 08/27] The step-up header test reads the shared constant through auth_models only (BACKLOG #3145) mypy strict on tests refused reading the constant off apiclient.client, which does not re-export it. The test now pins the constant's value in auth_models and that each of the three modules names it and no longer spells the literal. --- tests/test_oidc_json_step_up_hint.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_oidc_json_step_up_hint.py b/tests/test_oidc_json_step_up_hint.py index 248287cae4..6ce49a6c7c 100644 --- a/tests/test_oidc_json_step_up_hint.py +++ b/tests/test_oidc_json_step_up_hint.py @@ -411,15 +411,16 @@ def test_the_step_up_header_is_one_constant_on_both_sides() -> None: """Vault BACKLOG #3145 item 6: the gates, the two route refusals and the FastAPI-free client read one spelling, from the auth wire models, rather than four copies of a literal.""" from messagefoundry.api import auth_models - from messagefoundry.apiclient import client as apiclient assert _STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" - assert apiclient.STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER - assert apiclient.STEP_UP_VIA_IDP is auth_models.STEP_UP_VIA_IDP == "idp" + assert auth_models.STEP_UP_VIA_IDP == "idp" + # The client's use of the constant is behavioural: the replay tests above raise + # IdpStepUpRequired on the gate's own header. This pins that no copy of the literal is back. package = Path(auth_models.__file__).parent.parent for module in ("api/auth_routes.py", "apiclient/client.py", "api/security.py"): text = (package / module).read_text(encoding="utf-8") assert '"X-Step-Up-Via"' not in text, f"{module} spells the header out again" + assert "STEP_UP_VIA_HEADER" in text, f"{module} no longer reads the shared constant" async def test_a_store_fault_reading_the_mechanism_keeps_the_403( From 6fcd43b48f593e3738cc9dd74fce222dc9133226 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 18:14:49 -0500 Subject: [PATCH 09/27] The API client's sign-in and queued elevations wait 30 seconds (BACKLOG #3260 #3145 #2950) Item #2950. EngineClient.login, reauth and verify_mfa pass timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), a named 30-second constant matching the IDE's SIGN_IN_TIMEOUT_MS, the way integrity_check passes its own. reauth queues behind the per-account re-proof lock and verify_mfa behind the credential lock, and both rotate the token, so they get the same timeout. The engine's queue (keyed_lock.hold_keyed_lock) has no bound, so no value covers every wait: the login docstring says a lost reply is rarer, still possible, and why. Every other request keeps the 5-second default. apiclient stays FastAPI-free. New test tests/test_apiclient_sign_in_timeout.py reads request.extensions["timeout"] on an httpx.MockTransport: the three requests carry 30 s on every phase and /auth/me keeps 5 s; a second test reads the IDE's constant from ide/src/signInSession.ts. Proposed PR title: b257 D1+D2 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout Proposed vault banners: #3113 (D1 carry-over, round-two findings 1, 3, 5, 6): BUILT on engine branch b257-d2. The auth.mfa_enroll_refused row is written in a finally around the revoke, so no exception the revoke raises skips it, and aiosqlite's closed-connection ValueError is logged as a store refusal. The directory role notice says the open sessions were ended; the stale notifications.py comment and two SECURITY.md wording points are fixed. Round-two findings 2, 4 and 7 stay open. #3260: limb 2 BUILT on engine branch b257-d2, closing the row with limb 1 (engine PR 2190). revoke_own_session writes one auth.session_revoke_missed row per miss on both planes, through write_audit_soft, with the id cut to 12 characters and nothing else, so an unknown id and a foreign id get the same answer and the same row (owner ruling 2026-10-16, wiki note 20261016T211331989Z-vfzmx7). Register 1af question 4 is answered by this build, and the 1af entry needs a re-draft. #3145: BUILT on engine branch b257-d2. Items 4, 5 and 6(b)(c)(d) built, item 3 built for the opener's argument assembly; items 1, 2, 6(a), the checks.py preamble half of 3, and 6's extra session read declined in one written line each (see the #3145 commit body). #2950: BUILT on engine branch b257-d2. login, reauth and verify_mfa wait 30 s (SIGN_IN_TIMEOUT_SECONDS, the IDE's value); every other request keeps 5 s. The per-account queue has no bound, so a lost reply is rarer, not impossible. --- changelog.d/2950-sign-in-timeout.changed.md | 6 ++ messagefoundry/apiclient/client.py | 48 ++++++++-- tests/test_apiclient_sign_in_timeout.py | 99 +++++++++++++++++++++ 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 changelog.d/2950-sign-in-timeout.changed.md create mode 100644 tests/test_apiclient_sign_in_timeout.py diff --git a/changelog.d/2950-sign-in-timeout.changed.md b/changelog.d/2950-sign-in-timeout.changed.md new file mode 100644 index 0000000000..9bc2244f17 --- /dev/null +++ b/changelog.d/2950-sign-in-timeout.changed.md @@ -0,0 +1,6 @@ +- **The API client waits longer for a sign-in.** `EngineClient.login`, `reauth` and `verify_mfa` + now wait up to 30 seconds per phase, matching the IDE's sign-in timeout, instead of the client's + 5 second default. Each can wait in the engine's per-account queue, and a sign-in the client gave + up on could still end the session it replaced, leaving the client signed out. The longer wait + makes that rarer, not impossible, because the queue has no bound. Every other request keeps the + client's timeout. (`vault BACKLOG #2950`) diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index c5eeeff6a9..7c78e88710 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -135,6 +135,15 @@ # handlers by test_apiclient_approval_hold.py, which reads the code back rather than trusting 202. _HTTP_PENDING_APPROVAL = 202 +#: How long a request that can wait in the engine's per-account queue may go silent, in seconds +#: (vault BACKLOG #2950): the sign-in, and the two elevations that also rotate the token and queue +#: on that account, ``reauth`` behind the re-proof lock and ``verify_mfa`` behind the credential +#: lock. Matches the IDE's ``SIGN_IN_TIMEOUT_MS`` (``ide/src/signInSession.ts``). The queue has no +#: bound (``keyed_lock.hold_keyed_lock`` takes no timeout), so no value covers every wait: this +#: makes a lost reply rarer than the 5 s default did, and does not rule one out. Every other +#: request keeps the client's own timeout. +SIGN_IN_TIMEOUT_SECONDS = 30.0 + class ApiError(RuntimeError): """An API call failed (transport error, a non-2xx response, or an undecodable 2xx body).""" @@ -1124,7 +1133,17 @@ def reauth(self, password: str, *, purpose: str | None = None) -> None: body: dict[str, str] = {"password": password} if action is not None: body["purpose"] = action - self._adopt_rotated(self._request("POST", "/me/reauth", json=body, _allow_step_up=False)) + # Vault BACKLOG #2950: it rotates the token and queues behind the account's re-proof lock, + # so it gets the sign-in's timeout (see SIGN_IN_TIMEOUT_SECONDS). + self._adopt_rotated( + self._request( + "POST", + "/me/reauth", + json=body, + _allow_step_up=False, + timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + ) + ) def set_mfa_handler(self, handler: Callable[[], bool] | None) -> None: """Register the callback invoked when the engine demands a second factor (403 + @@ -1166,8 +1185,16 @@ def verify_mfa(self, code: str) -> None: for an ``oidc`` session that already met its factor, whose code was not checked either (BACKLOG #2142). Does not itself trigger the MFA handler.""" self._refuse_credential_on_cleartext("a second factor") + # Vault BACKLOG #2950: it rotates the token and queues behind the account's credential + # lock, so it gets the sign-in's timeout (see SIGN_IN_TIMEOUT_SECONDS). self._adopt_rotated( - self._request("POST", "/auth/mfa-verify", json={"code": code}, _allow_mfa=False) + self._request( + "POST", + "/auth/mfa-verify", + json={"code": code}, + _allow_mfa=False, + timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + ) ) def disable_mfa(self) -> None: @@ -1648,10 +1675,12 @@ def login( ends the old one, and the reply then fails to arrive, times out or will not decode, this client still holds the old token, which is dead, and never learns the new one. ``login`` raises :class:`ApiError` and every later call on the held token is refused - with a 401 until the caller signs in again. A timeout is the likely way in: the client - timeout (5 s per phase by default) also bounds this request, and a sign-in waiting in the - engine's per-account queue can take longer, while the engine goes on to finish it. - Before, the old token stayed live in that case. The new session nobody holds stays valid + with a 401 until the caller signs in again. A timeout is the likely way in: a sign-in + waiting in the engine's per-account queue can outlast the client, while the engine goes + on to finish it. This request therefore gets :data:`SIGN_IN_TIMEOUT_SECONDS` (30 s per + phase) rather than the client's 5 s default (vault BACKLOG #2950). That makes a lost + reply rarer, not impossible: the queue has no bound, so a long enough queue still + outlasts it. Before, the old token stayed live in that case. The new session nobody holds stays valid until it idles out or expires, as it did before. This is the accepted price of ending the old session ahead of the cap. It is not harmless. @@ -1663,7 +1692,12 @@ def login( prior = self._token if prior and self._token_cell.issued_here: body["supersedes"] = prior - result = _decode(self._request("POST", "/auth/login", json=body), LoginResponse) + result = _decode( + self._request( + "POST", "/auth/login", json=body, timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) + ), + LoginResponse, + ) self._hold_issued(result.token) self._user = result.user # A step-up purpose stashed for the old session must not bind the new one's next reauth(). diff --git a/tests/test_apiclient_sign_in_timeout.py b/tests/test_apiclient_sign_in_timeout.py new file mode 100644 index 0000000000..599bfeb376 --- /dev/null +++ b/tests/test_apiclient_sign_in_timeout.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 MessageFoundry Foundation, LLC and contributors +"""The sign-in and the two elevations that queue per account get a longer timeout (vault BACKLOG +#2950). + +``EngineClient.login`` names the token it replaces, and the engine ends that session inside the +sign-in (BACKLOG #2281). A sign-in waiting in the engine's per-account queue could outlast the +client's 5 s default, and a lost reply then leaves the client holding a token the engine ended. +``login``, ``reauth`` and ``verify_mfa`` now carry ``SIGN_IN_TIMEOUT_SECONDS``, which matches the +IDE's ``SIGN_IN_TIMEOUT_MS``; every other request keeps the client's own timeout. The queue has no +bound, so this makes the lost reply rarer, not impossible. + +The transport is an ``httpx.MockTransport``: what is pinned is the timeout each request carries, +read from ``request.extensions["timeout"]``, not what an engine does with it. +""" + +from __future__ import annotations + +from pathlib import Path + +import httpx + +from messagefoundry.apiclient import EngineClient +from messagefoundry.apiclient.client import SIGN_IN_TIMEOUT_SECONDS + +_BASE = "http://127.0.0.1:8765" +_USER = { + "user_id": "u1", + "username": "jdoe", + "auth_provider": "local", + "roles": ["viewer"], + "permissions": [], +} +#: Synthetic, low-entropy stand-ins for a session token. +_TOKEN = "t" * 43 +_ROTATED = "r" * 43 + + +def _reply(request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/auth/login": + body: object = {"token": _TOKEN, "user": _USER} + elif path in ("/me/reauth", "/auth/mfa-verify"): + body = {"detail": "re-verified", "token": _ROTATED} + else: + body = _USER + return httpx.Response(200, json=body) + + +def _recording_client() -> tuple[EngineClient, dict[str, dict[str, float | None]]]: + """An ``EngineClient`` over a mock transport, and the timeout each path's request carried.""" + seen: dict[str, dict[str, float | None]] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen[request.url.path] = dict(request.extensions["timeout"]) + return _reply(request) + + client = EngineClient(_BASE) + default = client._http.timeout + client._http.close() + # The client's own timeout, as _open_transport sets it, so the default is the one under test. + client._http = httpx.Client( + base_url=_BASE, timeout=default, transport=httpx.MockTransport(handler) + ) + return client, seen + + +def _every_phase(seconds: float) -> dict[str, float | None]: + return {"connect": seconds, "read": seconds, "write": seconds, "pool": seconds} + + +def test_the_sign_in_and_the_queued_elevations_carry_the_longer_timeout() -> None: + """RED when: ``/auth/login``, ``/me/reauth`` or ``/auth/mfa-verify`` runs under the client's + 5 s default (vault BACKLOG #2950), or ``/auth/me`` loses that default.""" + client, seen = _recording_client() + try: + client.login("jdoe", "a-strong-test-passphrase") + client.reauth("a-strong-test-passphrase") + client.verify_mfa("123456") + client.me() + finally: + client.close() + longer = _every_phase(SIGN_IN_TIMEOUT_SECONDS) + assert seen["/auth/login"] == longer + assert seen["/me/reauth"] == longer + assert seen["/auth/mfa-verify"] == longer + # The control: any other request keeps the client's default. + assert seen["/auth/me"] == _every_phase(5.0) + + +def test_the_timeout_matches_the_ides_sign_in_timeout() -> None: + """The two clients of the same queue wait the same time. The IDE's constant is read from its + source, so a change on either side reds here.""" + source = Path(__file__).resolve().parents[1] / "ide" / "src" / "signInSession.ts" + line = next( + ln for ln in source.read_text(encoding="utf-8").splitlines() if "SIGN_IN_TIMEOUT_MS =" in ln + ) + milliseconds = int(line.split("=", 1)[1].strip().rstrip(";").replace("_", "")) + assert milliseconds == SIGN_IN_TIMEOUT_SECONDS * 1000 From aaa9f56ebdf70ae18b33c53ae6849463bf505d73 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 18:35:10 -0500 Subject: [PATCH 10/27] Simplify pass: one store-fault set, one step-up header name, one bound refusal (BACKLOG #3145 #3113 #2950) - store/base.py gains a cached store_call_errors() (driver errors, RuntimeError, OSError, aiosqlite's closed-connection ValueError). api/security.py's step-up refusal catches it in place of its own copy, the refused enrolment's revoke catches it plus CipherError, and AuthService builds it at startup so the first fault does not import the server drivers on the event loop. - api/security.py drops its private _STEP_UP_VIA_HEADER alias for the shared auth_models constant; the test imports that, and its source scan is replaced by a pin of the wire spelling. - complete_oidc_step_up binds actor, client, return_to and flow once in a partial, so a new refusal arm cannot leave the address fields out. - apiclient builds its sign-in httpx.Timeout once at module level. Skipped: a shared test helper for swapping EngineClient's transport (four copies, three outside this diff); moving the sign-in timeout into _adopt_rotated (confirm_mfa rotates but holds no per-account lock on the engine, so it keeps the default). --- messagefoundry/api/security.py | 22 ++---- messagefoundry/apiclient/client.py | 9 +-- messagefoundry/auth/service.py | 108 ++++++++------------------- messagefoundry/store/base.py | 13 +++- tests/test_oidc_json_step_up_hint.py | 17 ++--- 5 files changed, 57 insertions(+), 112 deletions(-) diff --git a/messagefoundry/api/security.py b/messagefoundry/api/security.py index 5315f5a4b1..ba46bac395 100644 --- a/messagefoundry/api/security.py +++ b/messagefoundry/api/security.py @@ -18,7 +18,6 @@ import time from collections.abc import Awaitable, Callable, Coroutine, Mapping from dataclasses import dataclass -from functools import cache from typing import Any, get_args from fastapi import ( @@ -54,7 +53,7 @@ from messagefoundry.pipeline.alerts import AlertSink, LoggingAlertSink from messagefoundry.pipeline.cert_expiry import peer_cert_expiry from messagefoundry.redaction import safe_exc -from messagefoundry.store.base import store_driver_errors +from messagefoundry.store.base import store_call_errors from messagefoundry.store.store import UserRecord log = logging.getLogger(__name__) @@ -1129,19 +1128,6 @@ async def dependency(request: Request) -> Identity: " browser, not with a password: repeat the action in the web console, which opens /ui/reauth" " for it" ) -#: Set to ``idp`` on an ``oidc`` session's step-up refusal, so a client can branch on a header -#: rather than parse the detail. Absent on every other refusal. -_STEP_UP_VIA_HEADER = STEP_UP_VIA_HEADER - - -@cache -def _session_read_faults() -> tuple[type[Exception], ...]: - """What a refused session read raises: the drivers' errors, plus ``RuntimeError`` and - ``OSError`` as :func:`~messagefoundry.store.base.store_driver_errors` asks, and aiosqlite's - ``ValueError`` for a closed connection. A caller re-raises - :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` first, so a bug is not read as a fault. - Built on first use, because naming a server driver imports it.""" - return (RuntimeError, OSError, ValueError, *store_driver_errors()) async def _step_up_refusal( @@ -1160,7 +1146,7 @@ async def _step_up_refusal( via_idp = await auth.session_steps_up_at_idp(token) except AUDIT_WRITE_DEFECTS: raise - except _session_read_faults() as exc: + except store_call_errors() as exc: # Vault BACKLOG #3145 item 6: the read only picks the wording, and the gate has already # refused, so a store fault keeps the 403 rather than becoming a 500. It answers with the # password wording; an oidc client's POST /me/reauth then asks again and is told the IdP. @@ -1171,7 +1157,9 @@ async def _step_up_refusal( ) via_idp = False if via_idp: - headers[_STEP_UP_VIA_HEADER] = STEP_UP_VIA_IDP + # Set to ``idp`` on an ``oidc`` session's step-up refusal, so a client can branch on a + # header rather than parse the detail. Absent on every other refusal. + headers[STEP_UP_VIA_HEADER] = STEP_UP_VIA_IDP return HTTPException(status.HTTP_403_FORBIDDEN, _IDP_STEP_UP_DETAIL, headers=headers) return HTTPException(status.HTTP_403_FORBIDDEN, _STEP_UP_DETAIL, headers=headers) diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index 7c78e88710..726d37ef48 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -143,6 +143,7 @@ #: makes a lost reply rarer than the 5 s default did, and does not rule one out. Every other #: request keeps the client's own timeout. SIGN_IN_TIMEOUT_SECONDS = 30.0 +_SIGN_IN_TIMEOUT = httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) class ApiError(RuntimeError): @@ -1141,7 +1142,7 @@ def reauth(self, password: str, *, purpose: str | None = None) -> None: "/me/reauth", json=body, _allow_step_up=False, - timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + timeout=_SIGN_IN_TIMEOUT, ) ) @@ -1193,7 +1194,7 @@ def verify_mfa(self, code: str) -> None: "/auth/mfa-verify", json={"code": code}, _allow_mfa=False, - timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS), + timeout=_SIGN_IN_TIMEOUT, ) ) @@ -1693,9 +1694,7 @@ def login( if prior and self._token_cell.issued_here: body["supersedes"] = prior result = _decode( - self._request( - "POST", "/auth/login", json=body, timeout=httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) - ), + self._request("POST", "/auth/login", json=body, timeout=_SIGN_IN_TIMEOUT), LoginResponse, ) self._hold_issued(result.token) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 95faa6c05a..597f60571b 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -37,7 +37,7 @@ from contextvars import ContextVar from dataclasses import dataclass, replace from enum import Enum -from functools import cache +from functools import cache, partial from types import MappingProxyType from typing import Any, Final, Literal, TypeVar from uuid import uuid4 @@ -127,7 +127,7 @@ from messagefoundry.credential import constant_time_equal from messagefoundry.keyed_lock import KeyedLock as _KeyedLock from messagefoundry.keyed_lock import hold_keyed_lock as _hold_keyed_lock -from messagefoundry.store.base import AdminStore, store_driver_errors +from messagefoundry.store.base import AdminStore, store_call_errors, store_driver_errors from messagefoundry.store.crypto import MARKER_PREFIX, CipherError from messagefoundry.store.row_seal import ( ADSET_KIND, @@ -2602,6 +2602,9 @@ def __init__( # Built now, at startup, so a reconciler's first refused audit write does not import # the server drivers on the event loop (see `_audit_write_errors`). _audit_write_errors() + # The same for the soft catches that read store_call_errors (a refused enrolment's revoke, + # the step-up refusal's session read), whichever plane meets a store fault first. + store_call_errors() # Advisory, NON-STICKY federated-IdP health (ADR 0142 AC-8) — see the oidc_available docstring. self._oidc_unavailable_reason: str | None = None self._oidc_client_auth: oidc.ClientAuthentication | None = None @@ -4929,6 +4932,11 @@ async def complete_oidc_step_up( return await self._step_up_refused( FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow ) + # Every refusal below is made after the flow was redeemed, so each carries the address + # fields (vault BACKLOG #3145): bound once, so a new arm cannot leave them out. + refuse = partial( + self._step_up_refused, actor=actor, client=client, return_to=return_to, flow=flow + ) # BACKLOG #2301 (ASVS 2.4.2): max_age=0 and prompt=login make the person authenticate at the # IdP, so a step-up callback faster than a person can do that is refused. Before the code is # redeemed, so the refused flow spends nothing at the token endpoint. @@ -4937,58 +4945,40 @@ async def complete_oidc_step_up( # refusal waits for it below. With the list empty (the default) nothing changes here. arrived_too_early = self._oidc_callback_too_early(flow) if arrived_too_early and not self._oidc_callback_floor_exempt_values(): - return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(TOO_EARLY) try: principal_claims = await asyncio.to_thread( self._exchange_and_validate, code, flow, self._oidc_redirect_uri(public_origin) ) except oidc.ClaimsError as exc: # exc.reason is closed-set, so nothing IdP-influenced reaches the audit row. - return await self._step_up_refused( - exc.reason, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(exc.reason) except oidc.TokenRefusedError as exc: # The endpoint answered. As on the sign-in leg, that is not an outage (#1948), and the # status is what tells a junk code from a misconfiguration or a failing IdP. - return await self._step_up_refused( + return await refuse( "token_refused", - actor=actor, - client=client, - return_to=return_to, extra={"status": exc.status}, - flow=flow, ) except (OSError, ValueError, http.client.HTTPException) as exc: # The sign-in leg's outage arm, for the same reasons (#1948). Only the exception TYPE, # and a received status, are kept. if not _is_idp_transport_failure(exc): - return await self._step_up_refused( + return await refuse( IDP_ANSWER_UNUSABLE, - actor=actor, - client=client, - return_to=return_to, extra=_idp_error_fields(exc), - flow=flow, ) self.mark_oidc_unavailable(type(exc).__name__) - return await self._step_up_refused( + return await refuse( "idp_unavailable", - actor=actor, - client=client, - return_to=return_to, extra=_idp_error_fields(exc), - flow=flow, ) floor_exempt_amr = ( self._oidc_callback_floor_exemption(principal_claims.amr) if arrived_too_early else () ) if arrived_too_early and not floor_exempt_amr: # The opted-in site's too-early callback whose amr names no exempt value (BACKLOG #2388). - return await self._step_up_refused( - TOO_EARLY, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(TOO_EARLY) now = time.time() session = await self._load_session(token_hash) if ( @@ -5000,23 +4990,15 @@ async def complete_oidc_step_up( # stepped up, so the operator is not told "verified" and then signed out. or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): - return await self._step_up_refused( + return await refuse( "session_gone", - actor=actor, - client=client, - return_to=return_to, lost=True, - flow=flow, ) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: - return await self._step_up_refused( + return await refuse( "session_gone", - actor=actor, - client=client, - return_to=return_to, lost=True, - flow=flow, ) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, @@ -5030,18 +5012,14 @@ async def complete_oidc_step_up( # BEFORE both freshness tests (BACKLOG #2143), so another person's answer that passed # the claims ladder is filed as a subject mismatch whatever its auth_time. Both tests # below judge this session's identity, so neither is the right name for someone else. - return await self._step_up_refused( - STEP_UP_SUBJECT_MISMATCH, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(STEP_UP_SUBJECT_MISMATCH) # FRESHNESS, two tests, and both must pass. max_age=0 and prompt=login ask the IdP to # authenticate the user afresh, so a conforming IdP's auth_time postdates this request. # (a) Against our clock: auth_time is IdP clock and issued_at is ours, so the floor allows # the configured skew for an IdP clock that runs behind. skew = self._settings.oidc_clock_skew_seconds if flow.issued_at <= 0 or principal_claims.auth_time < flow.issued_at - skew: - return await self._step_up_refused( - STEP_UP_NOT_FRESH, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(STEP_UP_NOT_FRESH) # (b) Against the IdP's own clock (BACKLOG #2143): auth_time must be LATER than the one the # session holds, which is the sign-in's or the last step-up's. No skew applies, because # both values come from the IdP. This closes most of what (a) alone left: an IdP that @@ -5055,38 +5033,24 @@ async def complete_oidc_step_up( # row and the operator's text say which one refused. held_auth_time = session.idp_auth_time if held_auth_time is None: - return await self._step_up_refused( + return await refuse( STEP_UP_IDP_AUTH_TIME_MISSING, - actor=actor, - client=client, - return_to=return_to, - flow=flow, ) if principal_claims.auth_time <= held_auth_time: - return await self._step_up_refused( + return await refuse( STEP_UP_IDP_AUTH_TIME_NOT_LATER, - actor=actor, - client=client, - return_to=return_to, - flow=flow, ) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP # can still sign such a user in (sync lag, or its own credential store), so the engine asks # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: - return await self._step_up_refused( - "not_configured", actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse("not_configured") if not user.directory_object_id: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). - return await self._step_up_refused( + return await refuse( DIRECTORY_OBJECT_ID_MISSING, - actor=actor, - client=client, - return_to=return_to, - flow=flow, ) try: principal = await asyncio.to_thread( @@ -5096,20 +5060,14 @@ async def complete_oidc_step_up( _log.warning( "directory lookup failed during a federated step-up: %s", type(exc).__name__ ) - return await self._step_up_refused( - "directory_unavailable", actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse("directory_unavailable") if principal is None: - return await self._step_up_refused( - "not_in_directory", actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse("not_in_directory") # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. mismatch = _directory_answer_mismatch(principal, user.directory_object_id) if mismatch is not None: - return await self._step_up_refused( - mismatch, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(mismatch) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups # map to, the same test _directory_step_up_refusal applies on the TOTP and passkey legs # (#2240). Inline rather than through that helper, because its probe would change this @@ -5118,18 +5076,12 @@ async def complete_oidc_step_up( # as before, through the loader (BACKLOG #2592). fresh = await self._load_user(user.id) if fresh is None: - return await self._step_up_refused( + return await refuse( "session_gone", - actor=actor, - client=client, - return_to=return_to, lost=True, - flow=flow, ) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): - return await self._step_up_refused( - DIRECTORY_ROLES_DEMOTED, actor=actor, client=client, return_to=return_to, flow=flow - ) + return await refuse(DIRECTORY_ROLES_DEMOTED) purpose = flow.step_up_purpose # The password leg's three ORDER-CRITICAL steps (see :meth:`reauth`), against the hash. # (1) Every stamp against the OLD hash, re-anchoring the session to this client address. @@ -10249,7 +10201,7 @@ async def confirm_mfa_enrollment( log=_log, message="a refused TOTP enrolment for %s could not end the session it rotated", args=lambda: (scrub_log_argument(identity.username),), - errors=(*_audit_write_errors(), ValueError), + errors=(*store_call_errors(), CipherError), defects=AUDIT_WRITE_DEFECTS, ) finally: diff --git a/messagefoundry/store/base.py b/messagefoundry/store/base.py index a089041088..e0fd0c470a 100644 --- a/messagefoundry/store/base.py +++ b/messagefoundry/store/base.py @@ -27,7 +27,7 @@ import logging from collections.abc import AsyncIterator, Callable, Collection, Iterable, Mapping, Sequence from dataclasses import dataclass -from functools import partial +from functools import cache, partial from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable @@ -3260,6 +3260,17 @@ def store_connect_errors() -> tuple[type[Exception], ...]: return tuple(errors) +@cache +def store_call_errors() -> tuple[type[Exception], ...]: + """What a refused store call can raise, on any backend: :func:`store_driver_errors`, plus + ``RuntimeError`` and ``OSError`` as that function asks a caller to add, and ``ValueError``, + which aiosqlite raises for a closed connection before sqlite3 is reached. A caller that must not + read a bug as a refusal re-raises :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` + first. Cached, because naming a server driver imports it: build it at startup, so the first + refusal does not run that import on the event loop (vault BACKLOG #3145, #3113).""" + return (RuntimeError, OSError, ValueError, *store_driver_errors()) + + def store_open_errors() -> tuple[type[Exception], ...]: """:func:`store_driver_errors` and :func:`store_connect_errors`: at least the driver and connection errors a store open can raise, on any backend.""" diff --git a/tests/test_oidc_json_step_up_hint.py b/tests/test_oidc_json_step_up_hint.py index 6ce49a6c7c..00b65e3777 100644 --- a/tests/test_oidc_json_step_up_hint.py +++ b/tests/test_oidc_json_step_up_hint.py @@ -31,7 +31,8 @@ from fastapi import Depends, FastAPI from messagefoundry.api import create_app -from messagefoundry.api.security import _STEP_UP_VIA_HEADER, require_reauth_only +from messagefoundry.api.auth_models import STEP_UP_VIA_HEADER as _STEP_UP_VIA_HEADER +from messagefoundry.api.security import require_reauth_only from messagefoundry.apiclient import ApiError, EngineClient, IdpStepUpRequired from messagefoundry.auth import Role, totp from messagefoundry.auth.identity import ALL_CHANNELS, Identity @@ -407,20 +408,14 @@ def _prompt() -> bool: assert prompted == ["step-up"] -def test_the_step_up_header_is_one_constant_on_both_sides() -> None: +def test_the_shared_step_up_header_keeps_its_wire_spelling() -> None: """Vault BACKLOG #3145 item 6: the gates, the two route refusals and the FastAPI-free client - read one spelling, from the auth wire models, rather than four copies of a literal.""" + read one constant from the auth wire models. Its spelling is the wire contract, so a rename + there would break every older client; the replay tests above pin that each side uses it.""" from messagefoundry.api import auth_models - assert _STEP_UP_VIA_HEADER is auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" + assert auth_models.STEP_UP_VIA_HEADER == "X-Step-Up-Via" assert auth_models.STEP_UP_VIA_IDP == "idp" - # The client's use of the constant is behavioural: the replay tests above raise - # IdpStepUpRequired on the gate's own header. This pins that no copy of the literal is back. - package = Path(auth_models.__file__).parent.parent - for module in ("api/auth_routes.py", "apiclient/client.py", "api/security.py"): - text = (package / module).read_text(encoding="utf-8") - assert '"X-Step-Up-Via"' not in text, f"{module} spells the header out again" - assert "STEP_UP_VIA_HEADER" in text, f"{module} no longer reads the shared constant" async def test_a_store_fault_reading_the_mechanism_keeps_the_403( From 834697ee3cc123f49e7f9ff4438e2b4fdbe6ae31 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 19:20:31 -0500 Subject: [PATCH 11/27] Review round one on the D2 auth follow-ups (BACKLOG #3260 #3145 #2950) Confirmed and fixed: - The seam digest DOES record AuthService method signatures, so the required client keyword on revoke_own_session moved it. Regenerated ENGINE_UI_SEAM and the golden (0971e2440e4237c7), set the console's SUPPORTED_ENGINE_SEAMS, and corrected the _ui_seam note and the console changelog, which said the digest did not move. - confirm_mfa also rotates under the account's re-proof lock, and a lost reply there loses the recovery codes: it gets the sign-in timeout too. - The sign-in timeout raises the read phase only (httpx.Timeout(self._timeout, read=30)), so a GUI-thread caller still learns of an unreachable engine within the client's own bound. - The step-up refusal's store-fault catch logs with its traceback, so a defect raising ValueError or RuntimeError can be told from a fault. - The refused enrolment's row write is soft as well, so a store fault there cannot replace the answer or a propagating defect; the cancellation-after-commit limit is stated in the comment. - A sign-in flow refused as flow_purpose_mismatch no longer records a start_client from another ceremony. - The console's plain-text session id is copied into the miss row only when it is lower-case hex; anything else is recorded as malformed. - SECURITY.md says the miss row is soft-written; the IDE's step-up text says to repeat the action, as the engine's now does. - The refuse(...) calls lost their magic trailing commas. Open, out of this brief's scope: _audit_write_errors() still lacks aiosqlite's closed-connection ValueError for the reconciler's soft writes; that is the reconcile alert path, left to the D3 Builder. Proposed PR title: b257 D1+D2 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout Proposed vault banners: #3113 (D1 carry-over, round-two findings 1, 3, 5, 6): BUILT on engine branch b257-d2. The auth.mfa_enroll_refused row is written in a finally around the revoke, soft-written itself, so no exception the revoke raises skips it, and aiosqlite's closed-connection ValueError is logged as a store refusal. The directory role notice says the open sessions were ended; the stale notifications.py comment and two SECURITY.md wording points are fixed. Round-two findings 2, 4 and 7 stay open. #3260: limb 2 BUILT on engine branch b257-d2, closing the row with limb 1 (engine PR 2190). revoke_own_session writes one auth.session_revoke_missed row per miss on both planes, through write_audit_soft, with the id cut to 12 characters (a console id that is not hex is recorded as malformed) and nothing else, so an unknown id and a foreign id get the same answer and the same row (owner ruling 2026-10-16, wiki note 20261016T211331989Z-vfzmx7). The engine UI seam moved to 0971e2440e4237c7. Register 1af question 4 is answered by this build, and the 1af entry needs a re-draft. #3145: BUILT on engine branch b257-d2. Items 4, 5 and 6(b)(c)(d) built, item 3 built for the opener's argument assembly; items 1, 2, 6(a), the checks.py preamble half of 3, and 6's extra session read declined in one written line each (see the #3145 commit body). #2950: BUILT on engine branch b257-d2. login, reauth, verify_mfa and confirm_mfa wait 30 s for their reply (SIGN_IN_TIMEOUT_SECONDS, the IDE's value), read phase only; every other request keeps 5 s. The per-account queue has no bound, so a lost reply is rarer, not impossible. --- changelog.d/2950-sign-in-timeout.changed.md | 14 +-- docs/SECURITY.md | 8 +- ide/src/stepUp.ts | 2 +- messagefoundry/api/_ui_seam.py | 7 +- messagefoundry/api/security.py | 3 + messagefoundry/apiclient/client.py | 39 +++++--- messagefoundry/auth/service.py | 89 +++++++++---------- messagefoundry_webconsole/__init__.py | 2 +- .../messagefoundry-webconsole/CHANGELOG.md | 5 +- .../test_ui_session_revoke_miss_audit.py | 8 +- tests/golden/webconsole_seam.snapshot | 4 +- tests/test_apiclient_sign_in_timeout.py | 30 ++++--- 12 files changed, 118 insertions(+), 93 deletions(-) diff --git a/changelog.d/2950-sign-in-timeout.changed.md b/changelog.d/2950-sign-in-timeout.changed.md index 9bc2244f17..011693e040 100644 --- a/changelog.d/2950-sign-in-timeout.changed.md +++ b/changelog.d/2950-sign-in-timeout.changed.md @@ -1,6 +1,8 @@ -- **The API client waits longer for a sign-in.** `EngineClient.login`, `reauth` and `verify_mfa` - now wait up to 30 seconds per phase, matching the IDE's sign-in timeout, instead of the client's - 5 second default. Each can wait in the engine's per-account queue, and a sign-in the client gave - up on could still end the session it replaced, leaving the client signed out. The longer wait - makes that rarer, not impossible, because the queue has no bound. Every other request keeps the - client's timeout. (`vault BACKLOG #2950`) +- **The API client waits longer for a sign-in's reply.** `EngineClient.login`, `reauth`, + `verify_mfa` and `confirm_mfa` now wait up to 30 seconds for the engine's reply, matching the + IDE's sign-in timeout, instead of the client's 5 second default. Each can wait in the engine's + per-account queue, and a request the client gave up on could still end or rotate the session it + held, leaving the client signed out (and, for `confirm_mfa`, without its recovery codes). Only the + read phase is raised: an unreachable engine is still reported within the client's own timeout. + The longer wait makes a lost reply rarer, not impossible, because the queue has no bound. Every + other request keeps the client's timeout. (`vault BACKLOG #2950`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 3d4c48e79c..db47ce1ea3 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -756,7 +756,10 @@ stored id matters on the console. A self-service revoke of one session that ends `404` of `DELETE /me/sessions/{session_id}` and the console's "Nothing was revoked" notice, writes one `auth.session_revoke_missed` row (vault BACKLOG #3260, limb 2). It names the actor, the caller's address and the first 12 characters of the id, and nothing else: an unknown id and -another account's id get the same answer and the same row. +another account's id get the same answer and the same row. The console takes the id as plain +text, so an id that is not lower-case hex is recorded as `malformed` rather than copied. The +write is soft: if the audit log refuses it, the answer still stands and the loss is logged at +ERROR, so the row is the record of a miss only while the audit log accepts writes. **Delegated identity & admin device posture (#193 sibling; ASVS 13.2.1 / 13.3.2 / 8.4.2 — the delegation boundary).** Three controls whose enforcement is largely the deploying organization's to provide. @@ -3859,7 +3862,8 @@ token claim**. There is no new `auth_provider` value: a federated login resolves also carry `start_client`, the address the start leg staged on the flow, and `client_moved`, whether the callback came from another host by the new-address comparison, or null when either address is unknown (BACKLOG #2160, vault BACKLOG #3145). Neither is ever read from the token. A - refusal before the `state` check names neither. A move is recorded, never refused. Behind a proxy + refusal before the `state` check names neither, nor does a sign-in flow refused as + `flow_purpose_mismatch`, whose start was another ceremony. A move is recorded, never refused. Behind a proxy the engine does not trust, both legs carry the proxy's address, so `client_moved` reads false there. A cancel at the IdP returns to the same page. A callback without the flow's own `state` consumes nothing, and the flow waits for the real diff --git a/ide/src/stepUp.ts b/ide/src/stepUp.ts index c73683797e..f8e63787bb 100644 --- a/ide/src/stepUp.ts +++ b/ide/src/stepUp.ts @@ -30,7 +30,7 @@ export class IdpStepUpRequiredError extends Error { constructor() { super( "this session signed in through the identity provider, so it confirms a sensitive action " + - "there, in a browser: re-authenticate in the web console at /ui/reauth, then try again", + "there, in a browser: repeat the action in the web console, which opens /ui/reauth for it", ); this.name = "IdpStepUpRequiredError"; } diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index 27901b2e09..77f334a894 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -406,15 +406,16 @@ #: #: Vault BACKLOG #3260, limb 2: ``AuthService.revoke_own_session`` takes a required keyword #: ``client``, the address on its new ``auth.session_revoke_missed`` row and on its -#: ``auth.session_revoked`` row, and ``/ui/account/sessions/{id}/revoke`` passes it. The digest did -#: not move, for the same reason as #3259, and the same TypeError skew applies to that call. +#: ``auth.session_revoked`` row, and ``/ui/account/sessions/{id}/revoke`` passes it. Unlike the +#: ``AdminHandlers`` fields of #3259, an ``AuthService`` method is on the discovered surface with +#: its signature, so the digest moved and a skew in either direction is refused at mount. #: #: The digest below covers the surface DISCOVERED from the console's own imports and uses, which is #: strictly larger than the five hand-maintained tuples it replaced -- those had drifted, and the #: proof is that commit 40a4d5d9 added a REQUIRED ``UploadedFileList.scope`` field the console renders #: unconditionally while touching no seam file at all. Regenerate with #: ``python scripts/webconsole_seam_snapshot.py --write``; never hand-edit it to silence a gate. -ENGINE_UI_SEAM: str = "1b9da0aae75dbd52" +ENGINE_UI_SEAM: str = "0971e2440e4237c7" @dataclass(frozen=True, slots=True) diff --git a/messagefoundry/api/security.py b/messagefoundry/api/security.py index ba46bac395..954937f16e 100644 --- a/messagefoundry/api/security.py +++ b/messagefoundry/api/security.py @@ -1150,10 +1150,13 @@ async def _step_up_refusal( # Vault BACKLOG #3145 item 6: the read only picks the wording, and the gate has already # refused, so a store fault keeps the 403 rather than becoming a 500. It answers with the # password wording; an oidc client's POST /me/reauth then asks again and is told the IdP. + # With its traceback: ValueError and RuntimeError are also what a bug raises, and the log + # line is the only place a fault and a defect can be told apart. log.warning( "a step-up refusal could not read the session's sign-in mechanism (%s); it answers" " with the password wording", type(exc).__name__, + exc_info=True, ) via_idp = False if via_idp: diff --git a/messagefoundry/apiclient/client.py b/messagefoundry/apiclient/client.py index 726d37ef48..c578518240 100644 --- a/messagefoundry/apiclient/client.py +++ b/messagefoundry/apiclient/client.py @@ -135,15 +135,15 @@ # handlers by test_apiclient_approval_hold.py, which reads the code back rather than trusting 202. _HTTP_PENDING_APPROVAL = 202 -#: How long a request that can wait in the engine's per-account queue may go silent, in seconds -#: (vault BACKLOG #2950): the sign-in, and the two elevations that also rotate the token and queue -#: on that account, ``reauth`` behind the re-proof lock and ``verify_mfa`` behind the credential -#: lock. Matches the IDE's ``SIGN_IN_TIMEOUT_MS`` (``ide/src/signInSession.ts``). The queue has no -#: bound (``keyed_lock.hold_keyed_lock`` takes no timeout), so no value covers every wait: this -#: makes a lost reply rarer than the 5 s default did, and does not rule one out. Every other -#: request keeps the client's own timeout. +#: How long the reply to a request that can wait in the engine's per-account queue may take, in +#: seconds (vault BACKLOG #2950): the sign-in, and the elevations that rotate the token, which +#: queue on that account too: ``reauth``, ``verify_mfa`` and ``confirm_mfa`` (each rotation runs +#: under the account's re-proof lock, and ``verify_mfa`` also under its credential lock). Matches +#: the IDE's ``SIGN_IN_TIMEOUT_MS`` (``ide/src/signInSession.ts``). The queue has no bound +#: (``keyed_lock.hold_keyed_lock`` takes no timeout), so no value covers every wait: this makes a +#: lost reply rarer than the 5 s default did, and does not rule one out. It raises the read phase +#: only (:meth:`EngineClient._sign_in_timeout`); every other request keeps the client's timeout. SIGN_IN_TIMEOUT_SECONDS = 30.0 -_SIGN_IN_TIMEOUT = httpx.Timeout(SIGN_IN_TIMEOUT_SECONDS) class ApiError(RuntimeError): @@ -1084,6 +1084,13 @@ def _request( raise ApiError(_error_detail(response), status=response.status_code) return response + def _sign_in_timeout(self) -> httpx.Timeout: + """This client's timeout with only the READ phase raised to :data:`SIGN_IN_TIMEOUT_SECONDS` + (vault BACKLOG #2950). The engine's per-account queue is waited out after the request is + sent, so connect, write and pool keep the client's own bound: a caller on a GUI thread + still learns of an unreachable engine as fast as before.""" + return httpx.Timeout(self._timeout, read=SIGN_IN_TIMEOUT_SECONDS) + def _adopt_rotated(self, response: httpx.Response) -> None: """Adopt the re-keyed session token an elevation route hands back (ASVS 7.2.4). @@ -1142,7 +1149,7 @@ def reauth(self, password: str, *, purpose: str | None = None) -> None: "/me/reauth", json=body, _allow_step_up=False, - timeout=_SIGN_IN_TIMEOUT, + timeout=self._sign_in_timeout(), ) ) @@ -1175,7 +1182,11 @@ def confirm_mfa(self, code: str) -> list[str]: ``MfaConfirmResponse.token`` is a required ``str`` today, so a token-less body fails in ``_decode`` rather than clearing the session. That is a property of a different file, not of the rule, and the rule is the one thing all three sites must share.""" - response = self._request("POST", "/me/mfa/confirm", json={"code": code}) + # Vault BACKLOG #2950: the rotation queues behind the account's re-proof lock, and a lost + # reply here also loses the one-time recovery codes, so it gets the sign-in's timeout. + response = self._request( + "POST", "/me/mfa/confirm", json={"code": code}, timeout=self._sign_in_timeout() + ) self._adopt_rotated(response) return _decode(response, MfaConfirmResponse).recovery_codes @@ -1194,7 +1205,7 @@ def verify_mfa(self, code: str) -> None: "/auth/mfa-verify", json={"code": code}, _allow_mfa=False, - timeout=_SIGN_IN_TIMEOUT, + timeout=self._sign_in_timeout(), ) ) @@ -1678,8 +1689,8 @@ def login( ``login`` raises :class:`ApiError` and every later call on the held token is refused with a 401 until the caller signs in again. A timeout is the likely way in: a sign-in waiting in the engine's per-account queue can outlast the client, while the engine goes - on to finish it. This request therefore gets :data:`SIGN_IN_TIMEOUT_SECONDS` (30 s per - phase) rather than the client's 5 s default (vault BACKLOG #2950). That makes a lost + on to finish it. This request therefore waits :data:`SIGN_IN_TIMEOUT_SECONDS` (30 s) for + its reply rather than the client's 5 s default (vault BACKLOG #2950). That makes a lost reply rarer, not impossible: the queue has no bound, so a long enough queue still outlasts it. Before, the old token stayed live in that case. The new session nobody holds stays valid until it idles out or expires, as it did before. This is the accepted price of ending the @@ -1694,7 +1705,7 @@ def login( if prior and self._token_cell.issued_here: body["supersedes"] = prior result = _decode( - self._request("POST", "/auth/login", json=body, timeout=_SIGN_IN_TIMEOUT), + self._request("POST", "/auth/login", json=body, timeout=self._sign_in_timeout()), LoginResponse, ) self._hold_issued(result.token) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 597f60571b..7208b9a3e9 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -19,6 +19,7 @@ import logging import math import os +import re import secrets import time import unicodedata @@ -2031,6 +2032,8 @@ def _idp_error_fields(exc: BaseException) -> dict[str, object]: #: session writes when it ends nothing, because the id names no session or another account's. #: One action for both, and the same detail, so the row cannot tell the two apart either. SESSION_REVOKE_MISSED_ACTION: Final = "auth.session_revoke_missed" +#: What a session id can look like: a token hash, lower-case hex, cut to at most its full length. +_SESSION_ID_SHAPE: Final = re.compile(r"[0-9a-f]{1,64}") #: Vault BACKLOG #3260: the row an admin route writes when it refuses the caller's own account. #: Its detail's ``op`` names the route, one of :data:`SelfTargetOp`. @@ -4848,9 +4851,7 @@ async def abandon_oidc_step_up( return flow actor = await self._step_up_actor(flow) if flow.step_up_session_hash is None: - return await self._step_up_refused( - FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow - ) + return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) return await self._step_up_refused( reason, actor=actor, client=client, return_to=flow.return_to, flow=flow ) @@ -4912,8 +4913,9 @@ async def complete_oidc_step_up( The session is re-anchored to the callback's address when the callback has one. The ``auth.reauth`` row a completed step-up writes, including one whose rotation lost the session, also records the start leg's address as ``start_client`` and whether the two are - different hosts as ``client_moved`` (BACKLOG #2160). So does a refusal made after the flow - was redeemed (vault BACKLOG #3145); one made before carries neither. A move is recorded, + different hosts as ``client_moved`` (BACKLOG #2160). So does a refusal made after a + step-up flow was redeemed (vault BACKLOG #3145); :meth:`_step_up_refused` names the rows + that carry neither. A move is recorded, never refused. Behind a proxy the engine does not trust, both legs carry the proxy's address, so ``client_moved`` reads false there. @@ -4929,9 +4931,7 @@ async def complete_oidc_step_up( actor = await self._step_up_actor(flow) token_hash = flow.step_up_session_hash if token_hash is None: - return await self._step_up_refused( - FLOW_PURPOSE_MISMATCH, actor=actor, client=client, flow=flow - ) + return await self._step_up_refused(FLOW_PURPOSE_MISMATCH, actor=actor, client=client) # Every refusal below is made after the flow was redeemed, so each carries the address # fields (vault BACKLOG #3145): bound once, so a new arm cannot leave them out. refuse = partial( @@ -4956,23 +4956,14 @@ async def complete_oidc_step_up( except oidc.TokenRefusedError as exc: # The endpoint answered. As on the sign-in leg, that is not an outage (#1948), and the # status is what tells a junk code from a misconfiguration or a failing IdP. - return await refuse( - "token_refused", - extra={"status": exc.status}, - ) + return await refuse("token_refused", extra={"status": exc.status}) except (OSError, ValueError, http.client.HTTPException) as exc: # The sign-in leg's outage arm, for the same reasons (#1948). Only the exception TYPE, # and a received status, are kept. if not _is_idp_transport_failure(exc): - return await refuse( - IDP_ANSWER_UNUSABLE, - extra=_idp_error_fields(exc), - ) + return await refuse(IDP_ANSWER_UNUSABLE, extra=_idp_error_fields(exc)) self.mark_oidc_unavailable(type(exc).__name__) - return await refuse( - "idp_unavailable", - extra=_idp_error_fields(exc), - ) + return await refuse("idp_unavailable", extra=_idp_error_fields(exc)) floor_exempt_amr = ( self._oidc_callback_floor_exemption(principal_claims.amr) if arrived_too_early else () ) @@ -4990,16 +4981,10 @@ async def complete_oidc_step_up( # stepped up, so the operator is not told "verified" and then signed out. or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): - return await refuse( - "session_gone", - lost=True, - ) + return await refuse("session_gone", lost=True) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: - return await refuse( - "session_gone", - lost=True, - ) + return await refuse("session_gone", lost=True) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, principal_claims.subject, @@ -5033,13 +5018,9 @@ async def complete_oidc_step_up( # row and the operator's text say which one refused. held_auth_time = session.idp_auth_time if held_auth_time is None: - return await refuse( - STEP_UP_IDP_AUTH_TIME_MISSING, - ) + return await refuse(STEP_UP_IDP_AUTH_TIME_MISSING) if principal_claims.auth_time <= held_auth_time: - return await refuse( - STEP_UP_IDP_AUTH_TIME_NOT_LATER, - ) + return await refuse(STEP_UP_IDP_AUTH_TIME_NOT_LATER) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP # can still sign such a user in (sync lag, or its own credential store), so the engine asks @@ -5049,9 +5030,7 @@ async def complete_oidc_step_up( if not user.directory_object_id: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). - return await refuse( - DIRECTORY_OBJECT_ID_MISSING, - ) + return await refuse(DIRECTORY_OBJECT_ID_MISSING) try: principal = await asyncio.to_thread( self._ldap.resolve_principal, user.username, object_id=user.directory_object_id @@ -5076,10 +5055,7 @@ async def complete_oidc_step_up( # as before, through the loader (BACKLOG #2592). fresh = await self._load_user(user.id) if fresh is None: - return await refuse( - "session_gone", - lost=True, - ) + return await refuse("session_gone", lost=True) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): return await refuse(DIRECTORY_ROLES_DEMOTED) purpose = flow.step_up_purpose @@ -5171,9 +5147,10 @@ async def _step_up_refused( type (BACKLOG #1948). It never carries IdP text. ``flow`` adds ``start_client`` and ``client_moved``, as the success row carries them (vault - BACKLOG #3145). Only a refusal made after the flow was redeemed passes it: before ``state`` - matched, the callback is not shown to belong to the flow, and some refusals have no flow - at all, so those rows carry neither field.""" + BACKLOG #3145). Only a refusal made after a STEP-UP flow was redeemed passes it: before + ``state`` matched, the callback is not shown to belong to the flow; a sign-in flow refused + as ``flow_purpose_mismatch`` started another ceremony, so its address is not a step-up's; + and some refusals have no flow at all. Those rows carry neither field.""" # `extra` first, so it can never overwrite a closed-set field. detail: dict[str, object] = { **(extra or {}), @@ -8492,7 +8469,7 @@ async def revoke_own_session( or letting a user touch another's session. Audited. A miss writes :data:`SESSION_REVOKE_MISSED_ACTION` (vault BACKLOG #3260, limb 2; owner - ruling 2026-10-16). Its detail is the id's first 12 characters, as the success row's is, + ruling 2026-10-16). Its detail is the id's first 12 characters (or ``malformed``), as the success row's is, and nothing else: an unknown id and another account's id write the same row, so the trail records no ownership a caller could not already learn. The write is soft with no defect re-raised, for the reason ``api.auth_routes._refuse_if_self`` gives: a raise would replace @@ -8500,11 +8477,14 @@ async def revoke_own_session( :meth:`audit_self_target_refused`, because both planes call this from a request.""" session = await self._load_session(session_id) if session is None or session.user_id != identity.user_id: + # The console passes the id as plain text, so only an id-shaped one is copied into the + # trail; anything else is recorded as malformed, never as the caller's own characters. + shown = session_id[:12] if _SESSION_ID_SHAPE.fullmatch(session_id) else "malformed" await write_audit_soft( lambda: self._audit( SESSION_REVOKE_MISSED_ACTION, actor=actor, - detail=_json({"session": session_id[:12]}), + detail=_json({"session": shown}), client=client, ), log=_log, @@ -10192,8 +10172,12 @@ async def confirm_mfa_enrollment( # first, so no exception the revoke raises can skip it. ``session_ended`` records # what the revoke did. The token is not handed back either way, so the caller still # cannot use the session; a session the revoke left live expires on its own timers. - # A fault that also refuses the read and the audit write below still loses the row. + # The row's own write is soft too, so a store fault there is logged at ERROR rather + # than replacing the answer, or a defect already propagating. Known limit: a + # cancellation can land after the store committed the revoke, and the row then says + # ``session_ended: false`` for a session that did end. rotated = hash_token(elevation.token) + refused = (*store_call_errors(), CipherError) session_ended = False try: session_ended = await write_audit_soft( @@ -10201,11 +10185,18 @@ async def confirm_mfa_enrollment( log=_log, message="a refused TOTP enrolment for %s could not end the session it rotated", args=lambda: (scrub_log_argument(identity.username),), - errors=(*store_call_errors(), CipherError), + errors=refused, defects=AUDIT_WRITE_DEFECTS, ) finally: - await self._audit_enroll_refused(identity, session_ended, client=client) + await write_audit_soft( + lambda: self._audit_enroll_refused(identity, session_ended, client=client), + log=_log, + message="a refused TOTP enrolment for %s could not write its refusal row", + args=lambda: (scrub_log_argument(identity.username),), + errors=refused, + defects=AUDIT_WRITE_DEFECTS, + ) return Elevation(session_lost=True) await self._audit("auth.mfa_enrolled", actor=identity.username, client=client) # ADR 0197 Amendment A: enrolment now comes BEFORE the first rotation, so whoever intercepts diff --git a/messagefoundry_webconsole/__init__.py b/messagefoundry_webconsole/__init__.py index 3d9fc76524..bd54da3b42 100644 --- a/messagefoundry_webconsole/__init__.py +++ b/messagefoundry_webconsole/__init__.py @@ -44,7 +44,7 @@ # If cross-seam support is ever genuinely wanted, re-widen this set AND add the CI matrix that # installs the MIN and MAX supported engine builds — the claim and its test land together, or not # at all. -SUPPORTED_ENGINE_SEAMS: frozenset[str] = frozenset({"1b9da0aae75dbd52"}) +SUPPORTED_ENGINE_SEAMS: frozenset[str] = frozenset({"0971e2440e4237c7"}) #: The vendored static assets shipped in THIS wheel (mounted at /ui/static by :func:`mount_ui`). STATIC_DIR = Path(__file__).parent / "static" diff --git a/packaging/messagefoundry-webconsole/CHANGELOG.md b/packaging/messagefoundry-webconsole/CHANGELOG.md index 04cf29289c..2637da32db 100644 --- a/packaging/messagefoundry-webconsole/CHANGELOG.md +++ b/packaging/messagefoundry-webconsole/CHANGELOG.md @@ -95,8 +95,9 @@ this line.** - **A session revoke that ends nothing is audited, with your address.** The revoke button on `/ui/account/sessions` now passes the request's address to the engine, which writes one `auth.session_revoke_missed` row when the id names no session or another account's. The notice - is unchanged. `AuthService.revoke_own_session` takes a required keyword `client`, which the seam - digest does not record, so a skew raises TypeError on that call. (vault BACKLOG #3260) + is unchanged. `AuthService.revoke_own_session` takes a required keyword `client`; the seam + digest records that signature, so it moved, and a console and engine on either side of this + change refuse each other at mount. (vault BACKLOG #3260) - **The audit page shows where each entry came from.** `/ui/audit` has a Client column, which shows the address the engine recorded for the entry, or a dash when no client was in scope. An operator tracing a run of refused sign-ins can now see their source on the page, not only in diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py index 2bbfad27e1..b7b584f44d 100644 --- a/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py +++ b/packaging/messagefoundry-webconsole/tests/test_ui_session_revoke_miss_audit.py @@ -4,7 +4,7 @@ Owner ruling 2026-10-16: one ``auth.session_revoke_missed`` row per miss, with the id cut to its first 12 characters, and the answer unchanged and the same for an unknown id and another account's. -The console takes the id as plain text, so the cut also bounds what a caller can put in the row. +The console takes the id as plain text, so one that is not lower-case hex is recorded as malformed. The JSON plane's twin is ``tests/test_self_target_refusal_audit.py``. """ @@ -63,5 +63,7 @@ async def test_a_missed_console_revoke_writes_one_row_and_the_same_answer( assert len(rows) == 1, "one miss, one row" assert rows[0]["actor"] == "a" assert rows[0]["client"] == "127.0.0.1", "the miss row dropped the caller's address" - # The cut id and nothing else, so the row records no ownership either. - assert json.loads(str(rows[0]["detail"])) == {"session": target[:12]} + # The cut id and nothing else, so the row records no ownership either. Text that cannot be a + # session id is recorded as malformed, never copied. + shown = "malformed" if kind == "long" else target[:12] + assert json.loads(str(rows[0]["detail"])) == {"session": shown} diff --git a/tests/golden/webconsole_seam.snapshot b/tests/golden/webconsole_seam.snapshot index 41041dc563..904360b47c 100644 --- a/tests/golden/webconsole_seam.snapshot +++ b/tests/golden/webconsole_seam.snapshot @@ -7,7 +7,7 @@ # by hand (BACKLOG #1220) - so a newly rendered DTO is covered with nobody editing a list. ## ENGINE_UI_SEAM -1b9da0aae75dbd52 +0971e2440e4237c7 ## dataclass messagefoundry.api._ui_seam.UiDeps engine_seam @@ -195,7 +195,7 @@ oidc_flow_ttl_seconds: property -> 'int' password_change_owes_factor: (self, token: 'str | None') -> 'bool' reauth: (self, identity: 'Identity', password: 'str', *, token: 'str', client: 'str | None' = None, purpose: 'str | None' = None) -> 'Elevation' revoke_other_sessions: (self, identity: 'Identity', current_token_hash: 'str', *, actor: 'str') -> 'int' -revoke_own_session: (self, identity: 'Identity', session_id: 'str', *, actor: 'str') -> 'bool' +revoke_own_session: (self, identity: 'Identity', session_id: 'str', *, actor: 'str', client: 'str | None') -> 'bool' session_steps_up_at_idp: (self, token: 'str | None') -> 'bool' store: property -> 'AdminStore' suggested_notify_email: (profile_email: 'str | None') -> 'str | None' diff --git a/tests/test_apiclient_sign_in_timeout.py b/tests/test_apiclient_sign_in_timeout.py index 599bfeb376..7a8c74b158 100644 --- a/tests/test_apiclient_sign_in_timeout.py +++ b/tests/test_apiclient_sign_in_timeout.py @@ -6,9 +6,10 @@ ``EngineClient.login`` names the token it replaces, and the engine ends that session inside the sign-in (BACKLOG #2281). A sign-in waiting in the engine's per-account queue could outlast the client's 5 s default, and a lost reply then leaves the client holding a token the engine ended. -``login``, ``reauth`` and ``verify_mfa`` now carry ``SIGN_IN_TIMEOUT_SECONDS``, which matches the -IDE's ``SIGN_IN_TIMEOUT_MS``; every other request keeps the client's own timeout. The queue has no -bound, so this makes the lost reply rarer, not impossible. +``login``, ``reauth``, ``verify_mfa`` and ``confirm_mfa`` now wait ``SIGN_IN_TIMEOUT_SECONDS`` +for their reply, which matches the IDE's ``SIGN_IN_TIMEOUT_MS``; their other phases, and every +other request, keep the client's own timeout. The queue has no bound, so this makes the lost reply +rarer, not impossible. The transport is an ``httpx.MockTransport``: what is pinned is the timeout each request carries, read from ``request.extensions["timeout"]``, not what an engine does with it. @@ -42,6 +43,8 @@ def _reply(request: httpx.Request) -> httpx.Response: body: object = {"token": _TOKEN, "user": _USER} elif path in ("/me/reauth", "/auth/mfa-verify"): body = {"detail": "re-verified", "token": _ROTATED} + elif path == "/me/mfa/confirm": + body = {"recovery_codes": ["code-one"], "token": _ROTATED} else: body = _USER return httpx.Response(200, json=body) @@ -69,21 +72,28 @@ def _every_phase(seconds: float) -> dict[str, float | None]: return {"connect": seconds, "read": seconds, "write": seconds, "pool": seconds} -def test_the_sign_in_and_the_queued_elevations_carry_the_longer_timeout() -> None: - """RED when: ``/auth/login``, ``/me/reauth`` or ``/auth/mfa-verify`` runs under the client's - 5 s default (vault BACKLOG #2950), or ``/auth/me`` loses that default.""" +#: The requests that wait in the engine's per-account queue. +_QUEUED = ("/auth/login", "/me/reauth", "/auth/mfa-verify", "/me/mfa/confirm") + + +def test_the_sign_in_and_the_queued_elevations_wait_longer_for_their_reply() -> None: + """RED when: ``/auth/login``, ``/me/reauth``, ``/auth/mfa-verify`` or ``/me/mfa/confirm`` + waits only the client's 5 s default for its reply (vault BACKLOG #2950), when anything but + the read phase is raised, or when ``/auth/me`` loses that default.""" client, seen = _recording_client() try: client.login("jdoe", "a-strong-test-passphrase") client.reauth("a-strong-test-passphrase") client.verify_mfa("123456") + assert client.confirm_mfa("123456") == ["code-one"] client.me() finally: client.close() - longer = _every_phase(SIGN_IN_TIMEOUT_SECONDS) - assert seen["/auth/login"] == longer - assert seen["/me/reauth"] == longer - assert seen["/auth/mfa-verify"] == longer + # Only the read phase waits out the queue; connect, write and pool keep the client's bound, + # so an unreachable engine is still reported as fast as before. + longer = {**_every_phase(5.0), "read": SIGN_IN_TIMEOUT_SECONDS} + for path in _QUEUED: + assert seen[path] == longer, path # The control: any other request keeps the client's default. assert seen["/auth/me"] == _every_phase(5.0) From e6511df5cbe8f5980a69d8e00fba0889bfa0d153 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 20:07:43 -0500 Subject: [PATCH 12/27] The revocation-guard census lists the shared IdP opener wrapper as a builder (BACKLOG #3260 #3145 #2950) Review round two found tests/test_revocation_guard_census.py red on this branch: build_idp_opener_from_settings became a site the census did not list, and its two callers stopped counting as sites, so the census could no longer see AuthService.__init__'s tie to _refuse_idp_revocation. The wrapper is now in _BUILDERS, so both callers are sites again and the census passes (5 of 5). Round-two findings left open, for the PR body: a client timeout above 30 s shortens the read on the queued calls (use max); store_call_errors lacks pyodbc.Error for an unmapped SQLSTATE such as a deadlock; change_password also queues behind the re-proof lock and keeps 5 s; the store-fault fallback drops X-Step-Up-Via, so a client prompts a federated user for a password; the finally row write can replace a propagating CancelledError with a defect; the miss row's soft write catches Exception; three overlapping store-fault sets; the queue itself is unbounded (engine-side bound would close the lost reply); the 3260 changelog omits the malformed case; store_call_errors now imports the server drivers at every AuthService construction; _SESSION_ID_SHAPE admits short hex and duplicates DIGEST_ID_PATTERN; some added comment lines exceed 100 columns. Proposed PR title: b257 D1+D2 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout Proposed vault banners: #3113 (D1 carry-over, round-two findings 1, 3, 5, 6): BUILT on engine branch b257-d2. The auth.mfa_enroll_refused row is written in a finally around the revoke, soft-written itself, so no exception the revoke raises skips it, and aiosqlite's closed-connection ValueError is logged as a store refusal. The directory role notice says the open sessions were ended; the stale notifications.py comment and two SECURITY.md wording points are fixed. Round-two findings 2, 4 and 7 stay open. #3260: limb 2 BUILT on engine branch b257-d2, closing the row with limb 1 (engine PR 2190). revoke_own_session writes one auth.session_revoke_missed row per miss on both planes, through write_audit_soft, with the id cut to 12 characters (a console id that is not hex is recorded as malformed) and nothing else, so an unknown id and a foreign id get the same answer and the same row (owner ruling 2026-10-16, wiki note 20261016T211331989Z-vfzmx7). The engine UI seam moved to 0971e2440e4237c7. Register 1af question 4 is answered by this build, and the 1af entry needs a re-draft. #3145: BUILT on engine branch b257-d2. Items 4, 5 and 6(b)(c)(d) built, item 3 built for the opener's argument assembly; items 1, 2, 6(a), the checks.py preamble half of 3, and 6's extra session read declined in one written line each (see the #3145 commit body). #2950: BUILT on engine branch b257-d2. login, reauth, verify_mfa and confirm_mfa wait 30 s for their reply (SIGN_IN_TIMEOUT_SECONDS, the IDE's value), read phase only; every other request keeps 5 s. The per-account queue has no bound, so a lost reply is rarer, not impossible. --- tests/test_revocation_guard_census.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_revocation_guard_census.py b/tests/test_revocation_guard_census.py index 75153d4f51..15947c4fcf 100644 --- a/tests/test_revocation_guard_census.py +++ b/tests/test_revocation_guard_census.py @@ -69,6 +69,9 @@ "_verifying_context", "_build_tls_context", "build_idp_opener", + # The settings-reading wrapper both IdP opener sites call (vault BACKLOG #3145), so those + # two stay the sites and the wrapper hands its opener on, as build_idp_opener does. + "build_idp_opener_from_settings", "NarrowedTls", } ) From 2d1ddf145babaf92a19c9bc88de65ca811c3271b Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 21:38:16 -0500 Subject: [PATCH 13/27] A directory reconcile pass that newly grants Administrator raises administrator_granted (BACKLOG #3141) - AuthService takes an admin_grant_alert callable, the #3229 shape: the API lifespan builds it with pipeline.alerts.administrator_granted_alert and the marker, so auth/ imports no sink. - _apply_reconcile_revocation raises it right after the role write, on the sealed user.role_ids read at the top of the method, never the plan's roles. Only Administrator absent before and present after pages. Only a pass that did not abort applies a revocation. via = directory_reconcile. It pops the no_enabled_administrator cooldown, as intended. - Docs: alert_sinks comment, changelog.d/3229 and 2610 (2610 now states the server-error-after-role-write limit and the never-seen grant), new changelog.d/3141, SECURITY.md "A directory grant" bullets and the signal table, PHI.md alert_instance row, AlertSink docstring, the settings alert-type comment. - Item 3 of the row, from PR 2076 "Known, not fixed": the two low test findings are fixed (the console refused-after-sync tests assert the role was written; the caplog check filters on the logger name). Declined: two concurrent first sign-ins can each page once (a duplicate page is the safe direction); a future directory sign-in route must remember to call the alert (no route registry to enforce it against, and a census test is its own item). --- changelog.d/2610.security.md | 6 +- changelog.d/3141.security.md | 10 ++ changelog.d/3229.security.md | 4 +- docs/PHI.md | 2 +- docs/SECURITY.md | 23 ++- messagefoundry/api/app.py | 7 + messagefoundry/auth/service.py | 36 ++++ messagefoundry/config/settings.py | 3 +- messagefoundry/pipeline/alert_sinks.py | 8 +- messagefoundry/pipeline/alerts.py | 28 ++- .../tests/test_ui_directory_admin_alert.py | 9 + tests/test_directory_last_admin_alert.py | 165 +++++++++++++++++- tests/test_directory_sign_in_admin_alert.py | 5 +- 13 files changed, 282 insertions(+), 24 deletions(-) create mode 100644 changelog.d/3141.security.md diff --git a/changelog.d/2610.security.md b/changelog.d/2610.security.md index af9dcb5c82..b8df3fa63b 100644 --- a/changelog.d/2610.security.md +++ b/changelog.d/2610.security.md @@ -6,4 +6,8 @@ `user:`, with `via` naming the route and `granted_by` the fixed marker ``. A sign-in refused after the role was written still raises it, since a later sign-in gains nothing to page on. A sign-in that gains only other roles, or by an account that already held - Administrator, raises nothing. (`vault BACKLOG #2610`) + Administrator, raises nothing. At least two cases still raise no alert. A sign-in that fails with + a server error after the role write loses the page, because the retry gains nothing. And an + account that never signs in again and holds no live session raises nothing, because the engine + never sees the grant. A directory reconcile pass that applies the grant first raises the alert + itself (vault BACKLOG #3141). See docs/SECURITY.md, *A directory grant*. (`vault BACKLOG #2610`) diff --git a/changelog.d/3141.security.md b/changelog.d/3141.security.md new file mode 100644 index 0000000000..8ba45684c5 --- /dev/null +++ b/changelog.d/3141.security.md @@ -0,0 +1,10 @@ +- **A directory reconcile pass that newly grants Administrator now raises the + `administrator_granted` operator alert.** The session reconciler re-diffs a signed-in directory + account's roles and writes them. When that write gave the account Administrator, it raised only + `ad_session_revoked`, which does not name the role, and the account's next sign-in then gained + nothing, so no `administrator_granted` alert ever fired. The pass now raises it right after the + role write, keyed `user:`, with `via` set to `directory_reconcile` and `granted_by` the + fixed marker ``. An account that already held Administrator raises nothing. A pass the + mass-revoke breaker aborts writes no roles, so it raises nothing either. Like every + `administrator_granted` alert, it clears the account's `no_enabled_administrator` cooldown in the + engine process that raises it. (`vault BACKLOG #3141`) diff --git a/changelog.d/3229.security.md b/changelog.d/3229.security.md index 0aa2b60b62..aba01eb54b 100644 --- a/changelog.d/3229.security.md +++ b/changelog.d/3229.security.md @@ -7,8 +7,8 @@ `auth.no_enabled_administrator` audit row. With no enabled Administrator left, `provision-admin` runs again at the host. An `administrator_granted` alert for an account clears that account's `no_enabled_administrator` cooldown. It clears it only in the engine process that raises the - `administrator_granted` alert. A reconciler re-grant raises no `administrator_granted` alert, so - it leaves the cooldown in place. A directory disable or deletion of the last Administrator raises + `administrator_granted` alert. A reconciler re-grant raises `administrator_granted` too (vault + BACKLOG #3141), so it clears the cooldown the same way. A directory disable or deletion of the last Administrator raises no `no_enabled_administrator` alert, and `provision-admin` still refuses. For at least these cases and others that raise no `no_enabled_administrator` alert, see docs/SECURITY.md, *A directory role change can remove the last Administrator, and it alerts*. (`vault BACKLOG #3229`) diff --git a/docs/PHI.md b/docs/PHI.md index d3db7efce9..7ee01635a1 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1808,7 +1808,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 -- or a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index db47ce1ea3..c37922f1e8 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2402,7 +2402,7 @@ What the engine does instead is make the cheap routes loud. It refuses none of t | Signal | When | Where it goes | |---|---|---| | `approval.approver_provenance` audit row and `approval_approver_provenance` alert | A release goes ahead and the approver's account was created, had its password changed, or enrolled TOTP **after** the request was made. A resolution of an `interrupted` release does the same for the resolver's account (BACKLOG #2125) | Audit row against the approver or resolver, with their `client` address (ADR 0150). Its `act` field is `release` or `resolve`. Alert keyed `approval:` for a release and `approval::resolve` for a resolution, carrying the changed facts only | -| `administrator_granted` alert | `POST /users` creates an account with the Administrator role, `PUT /users/{id}/roles` adds it, `PUT /ad-group-map` newly maps a group to it, or a directory sign-in's role sync newly gives an account the role (vault BACKLOG #2610) | Alert keyed `user:` or `ad-group:`, naming the granting administrator, or `` for a directory sign-in | +| `administrator_granted` alert | `POST /users` creates an account with the Administrator role, `PUT /users/{id}/roles` adds it, `PUT /ad-group-map` newly maps a group to it, or a directory sign-in's role sync or a directory reconcile pass newly gives an account the role (vault BACKLOG #2610, #3141) | Alert keyed `user:` or `ad-group:`, naming the granting administrator, or `` for a directory sign-in or reconcile pass | | `client` on the `user.created` audit row | An account created through `POST /users` or `POST /users/directory` (BACKLOG #2021) | The creating administrator's address, like the approval rows | | `account_created` notice | An account created through `POST /users` or `POST /users/directory`, neither of which creates one without a notification address (BACKLOG #2018, #2021) | The new account's own notification address | @@ -2425,15 +2425,16 @@ with no page. `account_created` notice. - **A directory grant.** An account can get Administrator because the *directory* added it to a group already mapped to Administrator. The engine sees that grant in one of the two places below, - whichever comes first. A sign-in that sees it first raises `administrator_granted`, unless that - sign-in fails with a server error after the role write; the retry then gains nothing. A reconciler - pass that sees it first raises `ad_session_revoked`, which does not name the role. The later - sign-in then gains nothing, so it raises no `administrator_granted`. An account that never signs - in again and holds no live session raises nothing. So watch membership of the mapped group in the - directory itself. + whichever comes first. Either one raises `administrator_granted`, unless a sign-in that sees it + first fails with a server error after the role write; the retry then gains nothing. An account + that never signs in again and holds no live session raises nothing. So watch membership of the + mapped group in the directory itself. **CORRECTED 2026-10-01:** this read "raises no alert. The engine never sees that grant." **CORRECTED 2026-10-06:** this read "raises no `administrator_granted` alert", and the sign-in case below read "It raises no alert." Vault BACKLOG #2610 added the sign-in alert. + **CORRECTED 2026-10-16:** this read "A reconciler pass that sees it first raises + `ad_session_revoked`, which does not name the role. The later sign-in then gains nothing, so it + raises no `administrator_granted`." Vault BACKLOG #3141 added the reconciler's alert. - **At the account's next sign-in.** The engine writes an `auth.ad_roles_resynced` audit row with the old and new roles. It also sends a best-effort roles-changed notice to that account's own notification address, when the account has one and security notices are set @@ -2447,8 +2448,12 @@ with no page. [directory reconciler](#directory-session-reconciliation--propagating-an-ad-disable-adr-0079-mechanism-2) that completes stores the new roles and revokes the session. It writes `auth.ad_session_revoked` with the new roles, sends the same notice, and raises - `ad_session_revoked` with reason `roles_changed`. The next sign-in then writes no - `auth.ad_roles_resynced` row. That section lists when a pass revokes or alerts nothing. + `ad_session_revoked` with reason `roles_changed`. Right after the role write it raises + `administrator_granted`, keyed `user:`, with `granted_by` set to `` and + `via` set to `directory_reconcile`. That alert clears the account's `no_enabled_administrator` + cooldown, as a sign-in's does. An account that already held Administrator raises nothing. The + next sign-in then writes no `auth.ad_roles_resynced` row and raises no second + `administrator_granted`. That section lists when a pass revokes or alerts nothing. No pass runs at `[auth].ad_session_recheck_seconds = 0`. A pass probes at most `ad_session_recheck_max_users` accounts (200 by default), least recently probed first. When more directory accounts than that hold live sessions, an account can wait several passes. diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index d406f33509..93e51545aa 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -220,6 +220,7 @@ from messagefoundry.api.paging import page_total from messagefoundry.api.request_timeout import RequestTimeoutMiddleware from messagefoundry.api.security import ( + DIRECTORY_GRANTED_BY, AuthenticatedBeforeBodyRoute, ExportStopped, alert_sink_for, @@ -399,6 +400,7 @@ from messagefoundry.pipeline.alerts import ( AlertSink, LoggingAlertSink, + administrator_granted_alert, authority_seal_alert, no_enabled_administrator_alert, store_cipher_refusal_forwarder, @@ -10360,6 +10362,11 @@ async def _audit_convergence_reload(outcome: ReloadOutcome) -> None: # Vault BACKLOG #3229: a directory role change that leaves no enabled # Administrator is applied, and raises no_enabled_administrator here. admin_loss_alert=no_enabled_administrator_alert(notifier or LoggingAlertSink()), + # Vault BACKLOG #3141: a reconcile pass that newly grants Administrator raises + # administrator_granted here, as a directory sign-in does in its route. + admin_grant_alert=administrator_granted_alert( + notifier or LoggingAlertSink(), granted_by=DIRECTORY_GRANTED_BY + ), ) # Started only once the service is built, so a constructor refusal starts no task. if security_notifier is not None: diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 7208b9a3e9..3e74856870 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -2356,6 +2356,7 @@ def __init__( hop_posture: HopPosture | None = None, integrity_alert: Callable[[str], None] | None = None, admin_loss_alert: Callable[[str, str], None] | None = None, + admin_grant_alert: Callable[[str, str], None] | None = None, ) -> None: self._store = store self._settings = settings @@ -2365,6 +2366,9 @@ def __init__( # Vault BACKLOG #3229: called with (username, via) when a directory-driven role change leaves # no enabled Administrator. Injected by the API lifespan, as above. None logs only. self._admin_loss_alert = admin_loss_alert + # Vault BACKLOG #3141: called with (username, via) when a reconcile pass's role write newly + # gives an account Administrator. Injected by the API lifespan, as above. None logs only. + self._admin_grant_alert = admin_grant_alert # Session token-hash prefix -> monotonic time its seal failure was last reported, so one # bad row presented on every request writes one audit row and one alert per interval. self._seal_failures_reported = SealFailureThrottle() @@ -7332,6 +7336,12 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat await self._report_directory_admin_loss( user, before=user.role_ids, after=revocation.role_ids, via="ad-reconcile" ) + # Vault BACKLOG #3141. The gain side, from the same sealed read and at the same point, + # so a later step's failure cannot lose the page. The next sign-in finds the role + # already written and gains nothing, so this pass is the only chance to raise it. + self._report_directory_admin_grant( + user, before=user.role_ids, after=revocation.role_ids + ) revoked = await self._store.revoke_user_sessions(revocation.user_id) await self._audit( "auth.ad_session_revoked", @@ -7398,6 +7408,32 @@ async def _apply_reconcile_revocation(self, revocation: reconcile.SessionRevocat ) return True + def _report_directory_admin_grant( + self, user: UserRecord, *, before: Iterable[str], after: Iterable[str] + ) -> None: + """Raise ``administrator_granted`` when a reconcile pass's role write newly gave ``user`` + the Administrator role (vault BACKLOG #3141). + + The sign-in rule (vault BACKLOG #2610): only a role this write ADDED pages, so an account + that already held Administrator raises nothing. ``before`` is the account's roles as read + through its seal just before the write, never the plan's, which can be a pass old. Only an + applied revocation reaches here, and only a pass that did not abort applies one. The alert + pops the account's ``no_enabled_administrator`` cooldown in the sink, which is intended: + a later loss of the role pages again. + + Never raises, a defect in the callable included: it runs between the role write and the + session revocation, and a raise would skip the revocation (see + :meth:`_report_directory_admin_loss`).""" + admin = Role.ADMINISTRATOR.value + if admin in set(before) or admin not in set(after): + return + _log.warning("a directory reconcile pass gave an account the Administrator role") + if self._admin_grant_alert is not None: + try: + self._admin_grant_alert(user.username, "directory_reconcile") + except Exception: # noqa: BLE001 -- logged with its traceback; see the docstring + _log.exception("the administrator_granted alert callable raised") + async def _report_directory_admin_loss( self, user: UserRecord, diff --git a/messagefoundry/config/settings.py b/messagefoundry/config/settings.py index 20982ebc92..37ab6f318a 100644 --- a/messagefoundry/config/settings.py +++ b/messagefoundry/config/settings.py @@ -4813,7 +4813,8 @@ class AlertSeverity(str, Enum): # noqa: UP042 "approval_too_early", # BACKLOG #315: a release by an approver account changed after the request, and an # Administrator grant through the console API or, since vault BACKLOG #2610, by a directory - # sign-in's role sync (granted_by ``). + # sign-in's role sync, and since vault BACKLOG #3141 by a directory reconcile pass (both + # granted_by ``). "approval_approver_provenance", "administrator_granted", # vault BACKLOG #3229: a directory-driven role change left no enabled Administrator. Keyed diff --git a/messagefoundry/pipeline/alert_sinks.py b/messagefoundry/pipeline/alert_sinks.py index cba4f01a42..e7d19f3aad 100644 --- a/messagefoundry/pipeline/alert_sinks.py +++ b/messagefoundry/pipeline/alert_sinks.py @@ -1153,12 +1153,14 @@ def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None # BACKLOG #315: the Administrator role was granted. `user:` or `ad-group:` # stands in for "connection" (the key's grammar is on AlertSink). `granted_by` is the granting # administrator's operator account name, or the fixed marker `` when a directory - # sign-in's role sync made the grant (vault BACKLOG #2610). Never message content. + # sign-in's role sync or a reconcile pass made the grant (vault BACKLOG #2610, #3141). Never + # message content. # Vault BACKLOG #3229: every administrator_granted call pops the _last_sent key built from # the no_enabled_administrator alert type and the granted target, so a loss after an # administrator_granted re-grant is not held back by the earlier loss's cooldown. A - # reconciler re-grant calls no administrator_granted, so it pops nothing. The pop builds that key with its own - # f-string in `_emit`'s key format; tests/test_directory_last_admin_alert.py pins the match. + # reconciler re-grant raises administrator_granted too (vault BACKLOG #3141), so it pops the + # key the same way. The pop builds that key with its own f-string in `_emit`'s key format; + # tests/test_directory_last_admin_alert.py pins the match. # Only a `user:` target can match a loss key, so an `ad-group:` grant pops nothing. _last_sent is # per process, so this clears the cooldown in this process only. _occurrences and _suspended # are kept. Unlike crl_reload_applied, this resolves no open instance. diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index e1223e8337..29a07e6baf 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -505,12 +505,15 @@ def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None ``ad_group_map`` with ``name`` = ``ad-group:`` when a group newly maps to the role. A directory sign-in whose role sync newly grants the role (vault BACKLOG #2610) raises it with ``name`` = ``user:`` and ``via`` = ``directory_sign_in_negotiate``, - ``directory_sign_in_sso`` or ``directory_sign_in_oidc``, naming the route. + ``directory_sign_in_sso`` or ``directory_sign_in_oidc``, naming the route. A directory + reconcile pass whose role re-diff newly grants it (vault BACKLOG #3141) raises it with + ``name`` = ``user:`` and ``via`` = ``directory_reconcile``. Both keys are outside the connection-name grammar for the same reason as :meth:`approval_approver_provenance`. ``granted_by`` is the acting administrator's username, - or ```` on a directory sign-in, where no administrator acted. - No PHI. Emitted by the API's user-administration and directory sign-in routes, never from - ``auth/``.""" + or ```` on a directory sign-in or reconcile pass, where no administrator acted. + No PHI. Emitted by the API's user-administration and directory sign-in routes, and for the + reconciler through the callable :func:`administrator_granted_alert` builds, which the API + lifespan hands to the auth service, so ``auth/`` never imports a sink.""" ... def no_enabled_administrator(self, name: str, *, via: str) -> None: @@ -1293,6 +1296,23 @@ def _raise(username: str, via: str) -> None: return _raise +def administrator_granted_alert(sink: AlertSink, *, granted_by: str) -> Callable[[str, str], None]: + """An alert callable for :class:`~messagefoundry.auth.service.AuthService`: each call, with a + username and a ``via``, raises :meth:`AlertSink.administrator_granted` keyed ``user:`` + and naming ``granted_by`` (vault BACKLOG #3141). The API passes its directory marker, because + the service calls this only for a grant the directory made. + + Never raises: an alert failure must not change what a reconcile pass does.""" + + def _raise(username: str, via: str) -> None: + try: + sink.administrator_granted(f"user:{username}", via=via, granted_by=granted_by) + except Exception: # noqa: BLE001 — an alert-sink failure must never break a request + log.warning("the administrator_granted alert could not be delivered", exc_info=True) + + return _raise + + def alert_store_cipher_refusal(sink: AlertSink, table: str, column: str) -> None: """Raise the refusal alert for an unmarked value in ``table.column``: ``upload-cipher`` for the uploaded-file store, ``store-cipher`` for everything else. diff --git a/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py b/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py index f21e51f616..368b8e3c98 100644 --- a/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py +++ b/packaging/messagefoundry-webconsole/tests/test_ui_directory_admin_alert.py @@ -105,6 +105,13 @@ async def _service(engine: Engine, *, held: Role | None, bind: bool) -> AuthServ return service +async def _jdoe_holds_administrator(engine: Engine) -> bool: + """Whether the role write landed: a refused-after-sync test proves nothing without it.""" + jdoe = await engine.store.get_user_by_username("jdoe") + assert jdoe is not None + return Role.ADMINISTRATOR.value in await engine.store.get_user_role_ids(jdoe.id) + + def _client(engine: Engine, service: AuthService, sink: _Sink) -> httpx.AsyncClient: app = create_app(engine, auth=service, serve_ui=True, public_origin="https://ops.example") app.state.notifier = sink @@ -208,6 +215,7 @@ async def test_ui_sso_refused_after_the_role_write_still_raises_the_alert( page.""" sink = await _sso(engine, monkeypatch, held=Role.OPERATOR, bind_lands_mid_login=True) assert sink.events == [_grant("directory_sign_in_sso")] + assert await _jdoe_holds_administrator(engine) async def test_ui_oidc_that_newly_grants_administrator_raises_one_alert( @@ -231,3 +239,4 @@ async def test_ui_oidc_refused_after_the_role_write_still_raises_the_alert( failed, and a later sign-in gains nothing, so this refusal is the one chance to page.""" sink = await _oidc(engine, monkeypatch, held=None, bind_lands_mid_login=True) assert sink.events == [_grant("directory_sign_in_oidc")] + assert await _jdoe_holds_administrator(engine) diff --git a/tests/test_directory_last_admin_alert.py b/tests/test_directory_last_admin_alert.py index e07178b991..424d7c3cd2 100644 --- a/tests/test_directory_last_admin_alert.py +++ b/tests/test_directory_last_admin_alert.py @@ -26,6 +26,7 @@ import pytest +from messagefoundry.api.security import DIRECTORY_GRANTED_BY from messagefoundry.auth.ldap import AdPrincipal, DirectoryAnswer, DirectoryProbe from messagefoundry.auth.permissions import Role from messagefoundry.auth.service import NO_ENABLED_ADMINISTRATOR_ACTION, AuthService @@ -34,7 +35,11 @@ _ALERT_EVENT_TYPES, AuthSettings, ) -from messagefoundry.pipeline.alerts import LoggingAlertSink, no_enabled_administrator_alert +from messagefoundry.pipeline.alerts import ( + LoggingAlertSink, + administrator_granted_alert, + no_enabled_administrator_alert, +) from messagefoundry.store.store import MessageStore ADMINS = "cn=mf-admins,dc=test,dc=invalid" @@ -75,10 +80,12 @@ def probe_principal(self, username: str, *, object_id: str | None = None) -> Dir class _Rig: - def __init__(self, store: MessageStore) -> None: + def __init__(self, store: MessageStore, **overrides: Any) -> None: self.store = store self.ldap = _FakeLdap() self.alerts: list[tuple[str, str]] = [] + #: What the vault BACKLOG #3141 grant callable was handed, ``(username, via)``. + self.grants: list[tuple[str, str]] = [] settings = AuthSettings( ad_enabled=True, ad_server="ldaps://dc.test.invalid", @@ -86,12 +93,14 @@ def __init__(self, store: MessageStore) -> None: ad_bind_dn="CN=svc-mefor,OU=Service,DC=test,DC=invalid", ad_bind_password="synthetic", ad_session_recheck_seconds=60, + **overrides, ) self.service = AuthService( store, settings, ldap=self.ldap, # type: ignore[arg-type] admin_loss_alert=lambda username, via: self.alerts.append((username, via)), + admin_grant_alert=lambda username, via: self.grants.append((username, via)), ) async def start(self) -> None: @@ -322,3 +331,155 @@ async def test_a_loss_after_a_regrant_is_not_held_back_by_the_cooldown() -> None losses = [e["connection"] for e in transport.events if e["type"] == "no_enabled_administrator"] assert losses.count("user:jdoe") == 2 assert losses.count("user:asmith") == 1 + + +# --- vault BACKLOG #3141: a reconcile pass that newly grants Administrator pages ---------------- +# +# The reconciler's role re-diff can give an account Administrator. It raised only +# ad_session_revoked, and the account's next sign-in then found the role already written and +# gained nothing, so no administrator_granted ever fired. The pass now raises it right after the +# role write, judged on the roles read through the account's seal at apply time. + +#: The ``via`` a reconcile pass's grant reports. +RECONCILE_VIA = "directory_reconcile" + + +async def test_a_reconcile_pass_that_newly_grants_administrator_pages_once(rig: _Rig) -> None: + """RED before #3141: the pass raised no grant, and the next sign-in gains nothing.""" + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", ADMINS, via="ad-reconcile") + + assert await rig.roles("jdoe") == [Role.ADMINISTRATOR.value] + assert rig.grants == [("jdoe", RECONCILE_VIA)] + assert rig.alerts == [] + # The pass after it finds nothing to change, so it pages nothing more. + assert (await rig.service.reconcile_directory_sessions()).revocations == () + assert rig.grants == [("jdoe", RECONCILE_VIA)] + + +async def test_a_reconcile_change_to_an_account_that_held_administrator_is_quiet( + rig: _Rig, +) -> None: + """Negative control: Administrator held before the write, so nothing was granted.""" + rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", ADMINS, OPERATORS, via="ad-reconcile") + + assert sorted(await rig.roles("jdoe")) == sorted( + [Role.ADMINISTRATOR.value, Role.OPERATOR.value] + ) + assert rig.grants == [] + + +async def test_a_reconcile_gain_of_another_role_only_is_quiet(rig: _Rig) -> None: + """Negative control: a role was gained, but not Administrator.""" + rig.ldap.present["jdoe"] = _principal("jdoe") + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", OPERATORS, via="ad-reconcile") + + assert await rig.roles("jdoe") == [Role.OPERATOR.value] + assert rig.grants == [] + + +async def test_the_grant_is_judged_on_the_roles_read_at_apply_time_not_the_plans( + rig: _Rig, monkeypatch: pytest.MonkeyPatch +) -> None: + """A sign-in that lands between the pass's read and its write has already written + Administrator, and paged for it. The write then adds nothing, so the pass must not page again. + RED when the before-set is taken from the plan, which read the roles before that sign-in.""" + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) + real = rig.service._apply_reconcile_revocation + + async def a_sign_in_lands_first(revocation: Any) -> bool: + await rig.store.set_user_roles( + revocation.user_id, [Role.ADMINISTRATOR.value], assigned_by="ad-sync" + ) + return await real(revocation) + + monkeypatch.setattr(rig.service, "_apply_reconcile_revocation", a_sign_in_lands_first) + + plan = await rig.service.reconcile_directory_sessions() + + assert [(r.username, r.reason) for r in plan.revocations] == [("jdoe", "roles_changed")] + assert rig.grants == [] + + +async def test_a_pass_the_breaker_aborts_grants_nothing_and_pages_nothing() -> None: + """An aborted pass writes no roles, so there is no grant to page (ADR 0079 mechanism 2).""" + store = await MessageStore.open(":memory:") + try: + # One revocation of one probed account trips a breaker this tight. + rig = _Rig(store, ad_session_revoke_max=0, ad_session_revoke_max_fraction=0.01) + await rig.start() + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) + + plan = await rig.service.reconcile_directory_sessions() + + assert plan.aborted == "mass_revoke_breaker" + assert await rig.roles("jdoe") == [Role.OPERATOR.value] + assert rig.grants == [] + finally: + await store.close() + + +async def test_a_raising_grant_callable_still_ends_the_sessions(rig: _Rig) -> None: + """Only the production wrapper promises not to raise, so the service contains a raise: the + session revocation after the role write still runs.""" + + def raising(username: str, via: str) -> None: + raise RuntimeError("synthetic: the alert path is down") + + rig.service._admin_grant_alert = raising + rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) + await rig.sign_in("jdoe") + + await rig.directory_change("jdoe", ADMINS, via="ad-reconcile") + + assert await rig.roles("jdoe") == [Role.ADMINISTRATOR.value] + actions = [r["action"] for r in await rig.store.list_audit()] + assert "auth.ad_session_revoked" in actions + + +def test_the_grant_callable_keys_on_the_account_and_never_raises() -> None: + raised: list[tuple[str, str, str]] = [] + + class _Sink(LoggingAlertSink): + def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None: + raised.append((name, via, granted_by)) + + administrator_granted_alert(_Sink(), granted_by=DIRECTORY_GRANTED_BY)("jdoe", RECONCILE_VIA) + assert raised == [("user:jdoe", RECONCILE_VIA, DIRECTORY_GRANTED_BY)] + + class _Broken(LoggingAlertSink): + def administrator_granted(self, name: str, *, via: str, granted_by: str) -> None: + raise RuntimeError("synthetic: the sink is down") + + administrator_granted_alert(_Broken(), granted_by=DIRECTORY_GRANTED_BY)("jdoe", RECONCILE_VIA) + + +async def test_a_loss_after_a_reconciler_regrant_is_not_held_back_by_the_cooldown() -> None: + """The #3229 cooldown pop, through the reconciler's own grant callable: the second loss is + sent.""" + from messagefoundry.pipeline.alert_sinks import NotifierAlertSink + from tests.test_alert_sinks import _drain, _RecordingTransport + + transport = _RecordingTransport("webhook") + sink = NotifierAlertSink([transport]) # the default cooldown, far longer than this test + lose = no_enabled_administrator_alert(sink) + regrant = administrator_granted_alert(sink, granted_by=DIRECTORY_GRANTED_BY) + + lose("jdoe", "ad-reconcile") + regrant("jdoe", RECONCILE_VIA) + lose("jdoe", "ad-reconcile") + await _drain(sink) + + losses = [e["connection"] for e in transport.events if e["type"] == "no_enabled_administrator"] + assert losses.count("user:jdoe") == 2 diff --git a/tests/test_directory_sign_in_admin_alert.py b/tests/test_directory_sign_in_admin_alert.py index 9c306a5500..38a935db1e 100644 --- a/tests/test_directory_sign_in_admin_alert.py +++ b/tests/test_directory_sign_in_admin_alert.py @@ -245,4 +245,7 @@ async def test_a_sink_that_raises_does_not_break_the_sign_in( r = await _negotiate(c) assert r.status_code == 200, r.text assert sink.events == [_GRANT] - assert any("failed to emit" in rec.getMessage() for rec in caplog.records) + assert any( + rec.name == "messagefoundry.api.security" and "failed to emit" in rec.getMessage() + for rec in caplog.records + ) From f9df1af50d6e8b922b7e80bbdc3599915aca27ad Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:20:45 -0500 Subject: [PATCH 14/27] UNKEYED reconcile follow-ups: referrals count the probes asked, an outage still strikes (BACKLOG #3170) - Item 1: the referral's latched message, its auth.ad_reconcile_referred row (key renamed probed -> asked) and its ad_reconcile_aborted alert count plan.asked. The breaker-trip alert keeps plan.probed, its judged denominator. AlertSink docstring says which count is which. - Item 2: DECLINED, one line at the breaker call in plan_pass: exempting UNKEYED revocations changes what the mass-revoke breaker counts. - Item 3: plan_pass's two early returns (outage, referrals only) carry the UNKEYED rows' strikes, and only those: no outcomes, so the hold's record and judged_nothing are unchanged and the latch carries over. - Item 4: changelog.d/2434 names the auth.ad_reconcile_skipped probed -> asked rename. New changelog.d/3170. - Item 5: docs/PHI.md ad_session_revoked reasons gain directory_object_id_missing (as "at least"); the ad_reconcile_aborted count is described per source. - Item 6: the comment now in _directory_presence no longer says the reconciler probes an id-less row by name. - Item 7: the sign-in, step-up, re-proof, presence and bind refusals read _directory_key/_holds_directory_key; reason strings unchanged. _directory_key returns the id so mypy still narrows where it is sent. - Tests: an UNKEYED row is struck on a pass that judged nothing (three shapes, with an ordinary-pass control); a referral is counted over the probes asked, end to end and through _alert_reconcile_plan. --- changelog.d/2434.security.md | 4 +- changelog.d/3170.security.md | 9 ++++ docs/PHI.md | 2 +- docs/SECURITY.md | 5 +- messagefoundry/api/app.py | 4 +- messagefoundry/auth/reconcile.py | 20 ++++++- messagefoundry/auth/service.py | 57 ++++++++++++-------- messagefoundry/pipeline/alerts.py | 9 ++-- tests/test_ad_session_reconcile.py | 85 +++++++++++++++++++++++++++++- 9 files changed, 161 insertions(+), 34 deletions(-) create mode 100644 changelog.d/3170.security.md diff --git a/changelog.d/2434.security.md b/changelog.d/2434.security.md index 0a3e067971..fc09433c7d 100644 --- a/changelog.d/2434.security.md +++ b/changelog.d/2434.security.md @@ -3,7 +3,9 @@ directory row with no `objectGUID` and no federated binding is now judged without a lookup: it gets no roles written, and its sessions end at the strike threshold with the reason `directory_object_id_missing`, as an absent account's do. It never moves the ADR 0195 hold. No - sign-in or step-up admits such a row, so a session it holds is anomalous. + sign-in or step-up admits such a row, so a session it holds is anomalous. The count on the + outage's `auth.ad_reconcile_skipped` audit row is now the `asked` field, the lookups the pass + sent, and it leaves such a row out. It was the `probed` field. - **An empty password on the AD step-up re-bind is no longer counted as a wrong guess.** It is refused before the directory is asked, charged to neither the account lockout nor the session's re-proof budget, and audited on `auth.reauth` as `empty_password`. The caller sees a refused diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md new file mode 100644 index 0000000000..11e238675a --- /dev/null +++ b/changelog.d/3170.security.md @@ -0,0 +1,9 @@ +- **The session reconciler counts a referral over the lookups it sent, and strikes an id-less row + on every pass.** A directory row with no `objectGUID` is judged without a lookup, so no referral + can answer it. The referral's `auth.ad_reconcile_referred` audit row now carries the `asked` + field in place of `probed`, and that row, the referral's latched message and its + `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still + counts every account it judged. Such a row is also struck now on a pass that is a directory + outage or got only referrals, as on any other pass, so its sessions end at the strike threshold + however the directory answered. Its revocation still counts toward the mass-revoke breaker, as + before. (`vault BACKLOG #3170`) diff --git a/docs/PHI.md b/docs/PHI.md index 7ee01635a1..1ac085e254 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1808,7 +1808,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, the probed count and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with a `directory_absent`, `directory_disabled`, `directory_undetermined`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538; the payload is a fixed source label, `directory-reconciler` for the breaker and `directory-reconciler-referral` for a referral so each is its own instance, the closed-set reason slug `mass_revoke_breaker` or `directory_referral`, a count (the accounts the breaker judged, or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under either source, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index c37922f1e8..bae51cb518 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4114,7 +4114,10 @@ account's entry and write its roles onto the row. No sign-in or step-up admits s writes no roles, and it strikes and revokes as an absent account does, with the reason `directory_object_id_missing`. It is not a `userAccountControl` answer, so it never moves the ADR 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails -still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a +still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, +and one whose every lookup was referred, still strikes the row, because its answer never came from +the directory (BACKLOG #3170). A referral's `auth.ad_reconcile_referred` row and alert count only +the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less row, made before that refusal, is **never probed by name** (BACKLOG #2027). The pass skips the row and diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index 93e51545aa..8362ec7aa7 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -9512,7 +9512,9 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin sink.ad_reconcile_aborted( _REFERRAL_ALERT_SOURCE, reason=REFERRAL_ABORT, - probed=plan.probed, + # The probes ASKED (BACKLOG #3170): an UNKEYED row was never sent, so no referral + # could answer it. The breaker's count above stays its judged denominator. + probed=plan.asked, detail=auth.directory_reconcile_referral or REFERRAL_ABORT, ) if plan.hold: diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index 027892a3fb..017b88d42d 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -192,14 +192,17 @@ class ReconcilePlan: #: count``. A PRESENT probe maps to 0 (reset), and so does a HELD one (ADR 0195 rule item 7; #: ``held`` tells the two apart). An UNAVAILABLE or REFERRED probe is absent from this mapping, #: leaving whatever strike the user already carried untouched. Populated even on a breaker - #: abort — see :func:`plan_pass`. + #: abort — see :func:`plan_pass`. On an outage or a pass of referrals only it holds the UNKEYED + #: rows' strikes alone (BACKLOG #3170), since those were judged without the directory. strikes: Mapping[str, int] = field(default_factory=dict) #: Candidates this pass judged, asked or not. Excludes the per-pass budget remainder. An UNKEYED #: row counts here, because it can revoke. This is the breaker's denominator (``judged``). probed: int = 0 #: Probes this pass actually sent to the directory: ``probed`` less its UNKEYED rows (BACKLOG #: #2434). An outage and a pass of referrals only are judged over these, and the outage's log - #: line and the ``asked`` field of its ``auth.ad_reconcile_skipped`` row count these. + #: line and the ``asked`` field of its ``auth.ad_reconcile_skipped`` row count these. So do the + #: referral's latched message, the ``asked`` field of its ``auth.ad_reconcile_referred`` row + #: and its alert's count (BACKLOG #3170): a referral can only answer a probe that was sent. asked: int = 0 unavailable: int = 0 #: ``user_id`` of every probe the directory answered with a referral (BACKLOG #2538). Any one @@ -377,6 +380,14 @@ def plan_pass( # BACKLOG #2434. The caller reads a row it may not ask about as UNKEYED, so that outcome is # the one mark of "never asked": it says nothing about whether the directory answered. asked = sum(1 for p in probes if p.outcome is not ProbeOutcome.UNKEYED) + # BACKLOG #3170 item 3. An UNKEYED row was judged without the directory, so a pass that learned + # nothing from the directory still strikes it, as any other pass does. Only the strike: its + # outcome is not recorded here, so the hold's record, and so its carry-over, stays as it was. + unkeyed_strikes = { + p.user_id: prior_strikes.get(p.user_id, 0) + 1 + for p in probes + if p.outcome is ProbeOutcome.UNKEYED + } if referred and len(unavailable) + len(referred) == asked: # BACKLOG #2538. Nothing the pass asked was answered, as on an outage, so it judges nothing @@ -384,6 +395,7 @@ def plan_pass( # failures is not read as a plain outage, which pages nobody. A user search base in another # domain of the forest refers every probe, and this is the shape it takes. return ReconcilePlan( + strikes=unkeyed_strikes, probed=len(probes), asked=asked, unavailable=len(unavailable), @@ -399,6 +411,7 @@ def plan_pass( # zero-revocation pass. The hold is not judged on a pass that learned nothing, so its state # carries over. return ReconcilePlan( + strikes=unkeyed_strikes, probed=len(probes), asked=asked, unavailable=len(unavailable), @@ -501,6 +514,9 @@ def plan_pass( readable=readable, ) # Held and referred probes both stay in the breaker's denominator; ReconcilePlan.judged says why. + # UNKEYED revocations stay in its numerator, so while such rows hold sessions they push toward + # a trip on every pass. Exempting them is declined (BACKLOG #3170 item 2): it would change what + # the mass-revoke breaker counts, a security control, and that is not a follow-up's call. if breaker_tripped( revoke_count=len(revocations), probed=plan.judged, diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 3e74856870..f41b25bbe2 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -1586,17 +1586,25 @@ def _holds_federated_binding(user: UserRecord) -> bool: return user.oidc_issuer is not None or user.oidc_subject is not None +def _directory_key(user: UserRecord) -> str | None: + """The ``directory_object_id`` the engine may ask the directory about ``user`` by, or ``None`` + when the row carries none (an empty value counts as none). :func:`_holds_directory_key` states + the rule; a caller that goes on to send the id takes it from here, so the type narrows.""" + return user.directory_object_id or None + + def _holds_directory_key(user: UserRecord) -> bool: - """Whether the reconciler may ask the directory about ``user``: it carries a + """Whether the engine may ask the directory about ``user``: it carries a ``directory_object_id`` (BACKLOG #2434). - **The reconcile pass's one statement of the rule**; the sign-in, step-up and bind refusals - still test the column inline. A row without one is never asked about by name, so it is no - directory evidence: the pass reads it as UNKEYED without a lookup, and a trip or a referral - leaves it unmarked, because no pass can ever read it PRESENT. ``reconcile.plan_pass`` sees only - the UNKEYED outcome this produces, and counts what was asked from that. + **The one statement of the rule**: the reconcile pass and the sign-in, step-up and bind + refusals all read it (BACKLOG #3170 item 7). A row without one is never asked about by name, + so on the reconcile pass it is no directory evidence: the pass reads it as UNKEYED without a + lookup, and a trip or a referral leaves it unmarked, because no pass can ever read it PRESENT. + ``reconcile.plan_pass`` sees only the UNKEYED outcome this produces, and counts what was asked + from that. """ - return bool(user.directory_object_id) + return _directory_key(user) is not None def _holds_unkeyed_federated_binding(user: UserRecord) -> bool: @@ -4610,7 +4618,8 @@ async def _authenticate_oidc( username, "oidc", "local_account_conflict", client=client ) return LoginOutcome(ok=False, error="account conflict", reason="local_account_conflict") - if not bound.directory_object_id: + object_id = _directory_key(bound) + if object_id is None: # BACKLOG #2027 (ADR 0184 AC-5). A BOUND ROW WITH NO IMMUTABLE ID IS REFUSED, NOT # RE-RESOLVED BY NAME. The bind refuses such a row since BACKLOG #1143 slice C, so this # reaches only a binding written before that, or planted in the store. Its only @@ -4641,7 +4650,7 @@ async def _authenticate_oidc( # re-resolve is never keyed on the name alone (BACKLOG #2027). try: principal = await asyncio.to_thread( - self._ldap.resolve_principal, bound.username, object_id=bound.directory_object_id + self._ldap.resolve_principal, bound.username, object_id=object_id ) except LdapError as exc: await self._audit( @@ -4661,7 +4670,7 @@ async def _authenticate_oidc( ) # BACKLOG #3174: the answer must name the bound row's own directory object, by the test the # IdP step-up leg applies, before its groups or name are used. - mismatch = _directory_answer_mismatch(principal, bound.directory_object_id) + mismatch = _directory_answer_mismatch(principal, object_id) if mismatch is not None: await self._directory_reject_audit(username, "oidc", mismatch, client=client) return LoginOutcome(ok=False, error="federated sign-in failed", reason=mismatch) @@ -5031,13 +5040,14 @@ async def complete_oidc_step_up( # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: return await refuse("not_configured") - if not user.directory_object_id: + object_id = _directory_key(user) + if object_id is None: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). return await refuse(DIRECTORY_OBJECT_ID_MISSING) try: principal = await asyncio.to_thread( - self._ldap.resolve_principal, user.username, object_id=user.directory_object_id + self._ldap.resolve_principal, user.username, object_id=object_id ) except LdapError as exc: _log.warning( @@ -5048,7 +5058,7 @@ async def complete_oidc_step_up( return await refuse("not_in_directory") # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. - mismatch = _directory_answer_mismatch(principal, user.directory_object_id) + mismatch = _directory_answer_mismatch(principal, object_id) if mismatch is not None: return await refuse(mismatch) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups @@ -7251,7 +7261,7 @@ async def _record_reconcile_referral( self._reconcile_referral_alert = ( "LDAP referral: check [auth].ad_user_search_base and, for nested groups, " "[auth].ad_group_search_base; one names a base in another domain of the forest. " - f"{len(plan.referred)} of {plan.probed} signed-in directory account(s) probed were " + f"{len(plan.referred)} of {plan.asked} signed-in directory account(s) probed were " f"referred and left unjudged. {others} A referred account is never revoked while it " "is referred, because the engine does not follow referrals. Use a base in the bound " "domain controller's own domain, or a global catalog. The ERROR log line for each such " @@ -7273,7 +7283,9 @@ async def _record_reconcile_referral( "reason": reconcile.REFERRAL_ABORT, "referred": len(plan.referred), "unavailable": plan.unavailable, - "probed": plan.probed, + # Over the probes ASKED (BACKLOG #3170), as the outage's row is (#2434): an + # UNKEYED row was never sent, so no referral could answer it. + "asked": plan.asked, "aborted": plan.aborted, } ), @@ -8942,7 +8954,8 @@ async def _reproof_serialized( verdict: bool | None reason: str | None = None if directory: - if not user.directory_object_id: + object_id = _directory_key(user) + if object_id is None: # BACKLOG #2027, ADR 0184 AC-5. The re-bind finds its entry by the row's id, and a # row with none has only its name, which a directory may have reissued to someone # else whose password would then step this session up. Refused before the directory @@ -8950,9 +8963,7 @@ async def _reproof_serialized( # did not guess wrong. verdict, reason = None, DIRECTORY_OBJECT_ID_MISSING else: - rebind = await self._reauth_ad( - user.username, password, object_id=user.directory_object_id - ) + rebind = await self._reauth_ad(user.username, password, object_id=object_id) verdict, reason = rebind.verdict, rebind.reason else: verdict = user.password_hash is not None and await self._argon2( @@ -10655,11 +10666,11 @@ async def _directory_presence(self, user: UserRecord) -> reconcile.Probe | str: """ if self._ldap is None: return "not_configured" - if not user.directory_object_id: + if not _holds_directory_key(user): return DIRECTORY_OBJECT_ID_MISSING # The reconciler's own probe, so both ask the same question by the same key, off the loop. - # They differ on an id-less row only: this leg refuses it above, and the reconciler still - # probes an unbound one by name (BACKLOG #2027). + # Neither asks about an id-less row: this leg refuses it above, and the reconciler reads it + # as UNKEYED without a lookup (BACKLOG #2434, #3170 item 6). probe = await self._probe_principal(user) if probe.outcome is not reconcile.ProbeOutcome.PRESENT: return str(probe.outcome.value) @@ -12367,7 +12378,7 @@ async def audit_refused(reason: str, username: str, **extra: object) -> None: if user.auth_provider != AuthProvider.AD.value: await audit_refused("not_a_directory_account", user.username) raise ValueError("only a directory (AD) account can take a federated binding") - if not user.directory_object_id: + if not _holds_directory_key(user): # BACKLOG #1143 slice C: see DirectoryObjectIdMissing. After the no-op check, so a # legacy binding resubmitted as it stands gets the harmless answer above, while moving # it to another pair is still refused. Unbinding it stays open. diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index 29a07e6baf..b7462db6e9 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -542,10 +542,11 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s ``name`` labels the source: ``"directory-reconciler"`` for the breaker, and ``"directory-reconciler-referral"`` for a referral, so each is its own instance and - throttle. ``reason`` is one of those two closed-set slugs. ``probed`` is how many - principals the pass probed. ``detail`` is the operator-facing explanation the auth service - latches. No PHI. Emitted by the API-lifespan - reconciler task, never from ``auth/``. A whole-directory outage is NOT this event: it is + throttle. ``reason`` is one of those two closed-set slugs. ``probed`` is, for the breaker, + how many principals the pass judged, its denominator, and for a referral how many probes + the pass sent to the directory, which leaves out an id-less row (BACKLOG #3170). + ``detail`` is the operator-facing explanation the auth service latches. No PHI. Emitted by + the API-lifespan reconciler task, never from ``auth/``. A whole-directory outage is NOT this event: it is audited as ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine.""" ... diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 5dbe0199e8..4ea7e352a8 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -616,6 +616,38 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N assert alone.aborted is None and (alone.probed, alone.asked) == (1, 0) +@pytest.mark.parametrize( + "asked", + [ + [reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE)], + [reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED)], + [ + reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), + reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED), + ], + ], + ids=["outage", "referrals-only", "referrals-and-failures"], +) +def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( + asked: list[reconcile.Probe], +) -> None: + """BACKLOG #3170 item 3. An UNKEYED row is judged without the directory, so a pass that learned + nothing from it still strikes the row. Only the strike: the asked probes' strikes, the outcome + record the hold reads, and the hold's latch all stay as they were. + + RED before #3170: both early returns carried no strikes, so such a row went unstruck.""" + unkeyed = reconcile.Probe("k", "k", reconcile.ProbeOutcome.UNKEYED) + plan = _plan([*asked, unkeyed], prior_strikes={"k": 1, "a": 1, "r": 1}, latched=True) + assert plan.judged_nothing + assert plan.strikes == {"k": 2} + assert plan.outcomes == {} and plan.latched and not plan.hold + assert plan.revocations == () + # The control: the same row on an ordinary pass strikes the same way, and revokes there. + ordinary = _plan([unkeyed], prior_strikes={"k": 1}) + assert ordinary.strikes == {"k": 2} + assert [r.reason for r in ordinary.revocations] == ["directory_object_id_missing"] + + def test_breaker_ceiling_reports_the_larger_of_the_two_thresholds() -> None: assert reconcile.breaker_ceiling(probed=10, max_absolute=5, max_fraction=0.34) == 5 assert reconcile.breaker_ceiling(probed=300, max_absolute=5, max_fraction=0.34) == 102 @@ -872,7 +904,7 @@ async def test_a_referring_user_search_base_judges_nothing_and_pages_every_pass( "reason": "directory_referral", "referred": 2, "unavailable": 0, - "probed": 2, + "asked": 2, "aborted": "directory_referral", } assert len(referred) == 3 and all(d == expected for d in referred) @@ -1501,6 +1533,57 @@ async def test_a_referral_does_not_mark_an_id_less_row_it_can_never_confirm() -> await store.close() +async def test_a_referral_is_counted_over_the_probes_asked() -> None: + """BACKLOG #3170 item 1. An id-less row is never sent to the directory, so no referral could + answer it. The referral's row, its latched message and its alert count the probes asked, and + the row names the field ``asked``, as the outage's row does (#2434). + + RED before #3170: all three counted ``probed``, which takes in the id-less row.""" + from messagefoundry.api.app import _alert_reconcile_plan + from messagefoundry.pipeline.alerts import LoggingAlertSink + + store = await MessageStore.open(":memory:") + try: + ldap = _FakeLdap( + { + "alice": _principal("alice"), + "bob": _principal("bob"), + "jdoe": replace(_principal("jdoe"), directory_object_id=None), + } + ) + service = AuthService(store, _ad_settings(), ldap=ldap) # type: ignore[arg-type] + await service.initialize() + for name in ("alice", "bob", "jdoe"): + assert await _signed_in_ad_user(service, store, name) is not None + ldap.referring.add("alice") + + plan = await service.reconcile_directory_sessions() + + assert (plan.probed, plan.asked, len(plan.referred)) == (3, 2, 1) + [row] = _referred_rows(await store.list_audit()) + assert row["asked"] == 2 and "probed" not in row + message = service.directory_reconcile_referral + assert message is not None and "1 of 2 signed-in" in message + + counts: list[tuple[str, int]] = [] + + class _Sink(LoggingAlertSink): + def ad_reconcile_aborted( + self, name: str, *, reason: str, probed: int, detail: str + ) -> None: + counts.append((reason, probed)) + + _alert_reconcile_plan(plan, service, _Sink()) + assert counts == [("directory_referral", 2)] + # The breaker's page keeps its judged denominator, which counts the id-less row. + counts.clear() + tripped = replace(plan, aborted="mass_revoke_breaker") + _alert_reconcile_plan(tripped, service, _Sink()) + assert counts == [("mass_revoke_breaker", 3), ("directory_referral", 2)] + finally: + await store.close() + + async def test_a_breaker_trip_does_not_mark_an_id_less_row_or_forfeit_its_clear() -> None: """BACKLOG #2434, the breaker's half of the referral test above. A trip marks every candidate unconfirmed until a pass reads it PRESENT, and revoking a marked account forfeits the clear. An From 1027a1aae669ebc9324f1436691c89bc399f71ae Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:24:24 -0500 Subject: [PATCH 15/27] The reconcile-path store-fault set takes aiosqlite's closed-connection ValueError (b257 D2 carry-over) D2 carry-over. _audit_write_errors is now CipherError plus store_call_errors(), the one store-fault set D2 added, so a reconciler audit write against a closed aiosqlite connection (a bare ValueError, raised before sqlite3 is reached) is logged and passed over like any other refused write, instead of ending the pass. Callers that must not read a bug as a refusal still re-raise AUDIT_WRITE_DEFECTS first. The skip-report refusal test gains a closed-connection case, red on the previous set. --- messagefoundry/auth/service.py | 16 +++++++++------- tests/test_ad_session_reconcile.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index f41b25bbe2..c313a91341 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -128,7 +128,7 @@ from messagefoundry.credential import constant_time_equal from messagefoundry.keyed_lock import KeyedLock as _KeyedLock from messagefoundry.keyed_lock import hold_keyed_lock as _hold_keyed_lock -from messagefoundry.store.base import AdminStore, store_call_errors, store_driver_errors +from messagefoundry.store.base import AdminStore, store_call_errors from messagefoundry.store.crypto import MARKER_PREFIX, CipherError from messagefoundry.store.row_seal import ( ADSET_KIND, @@ -187,14 +187,16 @@ def _audit_write_errors() -> tuple[type[Exception], ...]: """What a refused audit write raises, at least on the store backends and key modes named here (BACKLOG #2137). - The drivers' own errors, plus the two that :func:`~messagefoundry.store.base.store_driver_errors` - asks a caller to add: ``RuntimeError`` for the engine's own refusals (the acquire timeout, a - keyless audit append) and ``OSError`` for a connection lost at the socket. And ``CipherError``, - which a ``vault_transit`` store raises when Transit cannot compute the audit-chain MAC (ADR - 0138). Built on first use, because naming a server driver imports it; an ``AuthService`` that + :func:`~messagefoundry.store.base.store_call_errors`: the drivers' own errors, ``RuntimeError`` + for the engine's own refusals (the acquire timeout, a keyless audit append), ``OSError`` for a + connection lost at the socket, and ``ValueError``, which aiosqlite raises for a closed + connection before sqlite3 is reached. And ``CipherError``, which a ``vault_transit`` store + raises when Transit cannot compute the audit-chain MAC (ADR 0138). A caller re-raises + :data:`~messagefoundry.audit_write.AUDIT_WRITE_DEFECTS` first where a bug must not read as a + refusal. Built on first use, because naming a server driver imports it; an ``AuthService`` that runs the reconciler builds it at construction, so that import never runs on the event loop in the middle of a store incident.""" - return (RuntimeError, OSError, CipherError, *store_driver_errors()) + return (CipherError, *store_call_errors()) def _warn_if_corpus_unreadable(path: str | None) -> None: diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 4ea7e352a8..93c752baec 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -1988,8 +1988,13 @@ async def _reported(store: MessageStore) -> list[str]: @pytest.mark.parametrize( "refusal", - [sqlite3.OperationalError("synthetic: disk I/O error"), CipherError("synthetic: Transit down")], - ids=["driver", "transit-mac"], + [ + sqlite3.OperationalError("synthetic: disk I/O error"), + CipherError("synthetic: Transit down"), + # aiosqlite's closed connection, raised before sqlite3 is reached (BACKLOG #3145 carry-over). + ValueError("no active connection"), + ], + ids=["driver", "transit-mac", "closed-connection"], ) async def test_a_refused_skip_report_logs_once_per_pass_and_tries_the_next_account_next( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, refusal: Exception @@ -1998,7 +2003,8 @@ async def test_a_refused_skip_report_logs_once_per_pass_and_tries_the_next_accou the first failure ended the pass; now the report stops at the first refusal, which also keeps a pool that times out each write to one timeout per pass. The account it refused goes last next pass, so the next account is tried. Both are reported once the - store accepts the write. A Transit outage refuses like a driver error (ADR 0138).""" + store accepts the write. A Transit outage refuses like a driver error (ADR 0138), and so does a + closed aiosqlite connection.""" store = await MessageStore.open(":memory:") try: service = await _two_unkeyed_bindings(store) From 7aae0c7c46b9d7d14e43498393bda4c7c5bd5e49 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:31:10 -0500 Subject: [PATCH 16/27] Simplify pass: the judged-nothing UNKEYED strikes are built only where they are used (BACKLOG #3170) plan_pass built the UNKEYED strike map on every pass and read it only in the two early returns. It is now a small helper, _unkeyed_strikes, called from those returns; the rule and its reason are stated there. Skipped from the pass: a shared factory for the two user-keyed alert callables (two copies only); converting the bind and presence refusals to _directory_key (they need no id); a RolesGained-based grant test in the reconciler (a two-line rule); reusing another file's test sink. --- messagefoundry/auth/reconcile.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index 017b88d42d..230586d7b3 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -346,6 +346,19 @@ def select_candidates( return ordered[: max(budget, 0)] +def _unkeyed_strikes(probes: list[Probe], prior_strikes: Mapping[str, int]) -> dict[str, int]: + """The strikes of a pass that judged nothing (BACKLOG #3170 item 3). + + An UNKEYED row was judged without the directory, so an outage or a pass of referrals only still + strikes it, as any other pass does. Only the strike: its outcome is not recorded, so the hold's + record, and so its carry-over, stays as it was, and an aborted pass revokes nothing.""" + return { + p.user_id: prior_strikes.get(p.user_id, 0) + 1 + for p in probes + if p.outcome is ProbeOutcome.UNKEYED + } + + def plan_pass( probes: Iterable[Probe], *, @@ -380,14 +393,6 @@ def plan_pass( # BACKLOG #2434. The caller reads a row it may not ask about as UNKEYED, so that outcome is # the one mark of "never asked": it says nothing about whether the directory answered. asked = sum(1 for p in probes if p.outcome is not ProbeOutcome.UNKEYED) - # BACKLOG #3170 item 3. An UNKEYED row was judged without the directory, so a pass that learned - # nothing from the directory still strikes it, as any other pass does. Only the strike: its - # outcome is not recorded here, so the hold's record, and so its carry-over, stays as it was. - unkeyed_strikes = { - p.user_id: prior_strikes.get(p.user_id, 0) + 1 - for p in probes - if p.outcome is ProbeOutcome.UNKEYED - } if referred and len(unavailable) + len(referred) == asked: # BACKLOG #2538. Nothing the pass asked was answered, as on an outage, so it judges nothing @@ -395,7 +400,7 @@ def plan_pass( # failures is not read as a plain outage, which pages nobody. A user search base in another # domain of the forest refers every probe, and this is the shape it takes. return ReconcilePlan( - strikes=unkeyed_strikes, + strikes=_unkeyed_strikes(probes, prior_strikes), probed=len(probes), asked=asked, unavailable=len(unavailable), @@ -411,7 +416,7 @@ def plan_pass( # zero-revocation pass. The hold is not judged on a pass that learned nothing, so its state # carries over. return ReconcilePlan( - strikes=unkeyed_strikes, + strikes=_unkeyed_strikes(probes, prior_strikes), probed=len(probes), asked=asked, unavailable=len(unavailable), From d7e3e9e4db1fa7118c184566897cff9d9d0b26a5 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 22:52:57 -0500 Subject: [PATCH 17/27] Review round one on the D3 reconcile follow-ups (BACKLOG #3141 #3170) - changelog.d/3170 and SECURITY.md: an outage or referral-only pass strikes an id-less row but revokes nothing, so its sessions end at the next pass that is not aborted, not at the threshold regardless. - SECURITY.md: a sign-in running at the same time as the pass can raise a second administrator_granted; a duplicate is the safe direction. - LoggingAlertSink's ad_reconcile_aborted line no longer calls the count the pass's size, since a referral's count is the probes asked. - Tests: the reconciler-grant test now proves the next sign-in gains nothing (the old second-pass check ran on no candidate); the reconciler cooldown-pop test gains the cross-account control. --- changelog.d/3170.security.md | 7 ++++--- docs/SECURITY.md | 7 +++++-- messagefoundry/pipeline/alerts.py | 3 ++- tests/test_directory_last_admin_alert.py | 12 ++++++++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 11e238675a..2795c080d7 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -4,6 +4,7 @@ field in place of `probed`, and that row, the referral's latched message and its `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or got only referrals, as on any other pass, so its sessions end at the strike threshold - however the directory answered. Its revocation still counts toward the mass-revoke breaker, as - before. (`vault BACKLOG #3170`) + outage or got only referrals, as on any other pass. Such a pass still revokes nothing, so the + row's sessions end at the first pass after it that is not aborted, and only at their own expiry + while every pass aborts. Its revocation still counts toward the mass-revoke breaker, as before. + (`vault BACKLOG #3170`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index bae51cb518..b042428a3d 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2453,7 +2453,9 @@ with no page. `via` set to `directory_reconcile`. That alert clears the account's `no_enabled_administrator` cooldown, as a sign-in's does. An account that already held Administrator raises nothing. The next sign-in then writes no `auth.ad_roles_resynced` row and raises no second - `administrator_granted`. That section lists when a pass revokes or alerts nothing. + `administrator_granted`. A sign-in that runs at the same time as the pass can still raise a + second one, since each judges against its own read of the roles; a duplicate page is the safe + direction. That section lists when a pass revokes or alerts nothing. No pass runs at `[auth].ad_session_recheck_seconds = 0`. A pass probes at most `ad_session_recheck_max_users` accounts (200 by default), least recently probed first. When more directory accounts than that hold live sessions, an account can wait several passes. @@ -4116,7 +4118,8 @@ writes no roles, and it strikes and revokes as an absent account does, with the 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, and one whose every lookup was referred, still strikes the row, because its answer never came from -the directory (BACKLOG #3170). A referral's `auth.ad_reconcile_referred` row and alert count only +the directory (BACKLOG #3170). Neither pass revokes it: that waits for the next pass that is not +aborted. A referral's `auth.ad_reconcile_referred` row and alert count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index b7462db6e9..a87aa46298 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -1088,7 +1088,8 @@ def no_enabled_administrator(self, name: str, *, via: str) -> None: def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: log.warning( - "ALERT ad_reconcile_aborted: %r left accounts unrevoked in a pass of %d principal(s) " + # The count is the breaker's judged total or a referral's probes asked (BACKLOG #3170). + "ALERT ad_reconcile_aborted: %r left accounts unrevoked, counting %d principal(s) " "(%s): %s", name, probed, diff --git a/tests/test_directory_last_admin_alert.py b/tests/test_directory_last_admin_alert.py index 424d7c3cd2..0631fd0f75 100644 --- a/tests/test_directory_last_admin_alert.py +++ b/tests/test_directory_last_admin_alert.py @@ -354,8 +354,11 @@ async def test_a_reconcile_pass_that_newly_grants_administrator_pages_once(rig: assert await rig.roles("jdoe") == [Role.ADMINISTRATOR.value] assert rig.grants == [("jdoe", RECONCILE_VIA)] assert rig.alerts == [] - # The pass after it finds nothing to change, so it pages nothing more. - assert (await rig.service.reconcile_directory_sessions()).revocations == () + # Why the pass must page: the account's next sign-in finds the role written and gains nothing. + outcome = await rig.service._complete_ad_login( + rig.ldap.present["jdoe"], None, mfa_verified=True + ) + assert outcome.ok and outcome.roles_gained is None assert rig.grants == [("jdoe", RECONCILE_VIA)] @@ -477,9 +480,14 @@ async def test_a_loss_after_a_reconciler_regrant_is_not_held_back_by_the_cooldow regrant = administrator_granted_alert(sink, granted_by=DIRECTORY_GRANTED_BY) lose("jdoe", "ad-reconcile") + # The control: a grant that cleared every account's cooldown would let asmith's second loss + # through and fail the count below. + lose("asmith", "ad-reconcile") regrant("jdoe", RECONCILE_VIA) lose("jdoe", "ad-reconcile") + lose("asmith", "ad-reconcile") await _drain(sink) losses = [e["connection"] for e in transport.events if e["type"] == "no_enabled_administrator"] assert losses.count("user:jdoe") == 2 + assert losses.count("user:asmith") == 1 From ed6efa27d6982a8a00a1f198a30e63cd543498c7 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Fri, 16 Oct 2026 23:11:51 -0500 Subject: [PATCH 18/27] Review round two: the docs stop over-claiming what an aborted pass does (BACKLOG #3141 #3170) Docs and docstrings only, from the second code-review round: - SECURITY.md's Fail-OPEN bullet and the referring-base paragraph, and reconcile.py's module and judged_nothing docstrings, no longer say an outage or referral-only pass strikes or judges nothing: it strikes an id-less row (BACKLOG #3170). - SECURITY.md and changelog.d/3170: an id-less row is revoked only by a pass that is not aborted, once its strikes reach the threshold and it is in that pass's probe budget. - SECURITY.md "A directory grant": the cases that raise no administrator_granted are "at least" these, adding a breaker-aborted pass and recheck_seconds = 0. Proposed PR title: Directory reconcile follow-ups: a reconciler grant pages administrator_granted, UNKEYED referral counts and outage strikes (BACKLOG #3141 #3170) Proposed vault banner, #3141: BUILT on engine branch b257-d3 (batch 257, D3). A reconcile pass whose role write newly gives an account Administrator raises administrator_granted (via directory_reconcile, granted_by ) right after the write, through an injected callable, judged on the sealed apply-time roles; an aborted pass writes nothing and pages nothing. It pops the no_enabled_administrator cooldown as intended. changelog.d/2610 now states the server-error-after-role-write limit. Item 3: the two low test findings fixed; concurrent first sign-ins paging twice and the future-route reminder declined in writing. Vault #3602 item 4 closes with this; item 8 (the reconciler-first timing race) closes too, since the reconciler-first case now pages. Proposed vault banner, #3170: BUILT on engine branch b257-d3 (batch 257, D3). Items 1, 3, 4, 5, 6, 7 fixed: the referral's message, audit row (field renamed to asked) and alert count plan.asked, the breaker alert keeps probed; outage and referral-only passes strike UNKEYED rows (strikes only); changelog 2434 names the rename; PHI.md lists directory_object_id_missing; the _directory_presence comment is corrected; the inline id checks read _directory_key/_holds_directory_key with reason strings unchanged. Item 2 DECLINED by Manager ruling: exempting UNKEYED revocations would change what the mass-revoke breaker counts. Items 1 and 3 have tests. --- changelog.d/3170.security.md | 6 +++--- docs/SECURITY.md | 19 +++++++++++-------- messagefoundry/auth/reconcile.py | 6 ++++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 2795c080d7..44397d0f6f 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -4,7 +4,7 @@ field in place of `probed`, and that row, the referral's latched message and its `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or got only referrals, as on any other pass. Such a pass still revokes nothing, so the - row's sessions end at the first pass after it that is not aborted, and only at their own expiry - while every pass aborts. Its revocation still counts toward the mass-revoke breaker, as before. + outage or got only referrals, as on any other pass. Such a pass still revokes nothing. Only a + pass that is not aborted can, once the row's strikes reach the threshold and the row is in that + pass's probe budget, so while every pass aborts its sessions last to their own expiry. Its revocation still counts toward the mass-revoke breaker, as before. (`vault BACKLOG #3170`) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index b042428a3d..731c76e824 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -2425,10 +2425,12 @@ with no page. `account_created` notice. - **A directory grant.** An account can get Administrator because the *directory* added it to a group already mapped to Administrator. The engine sees that grant in one of the two places below, - whichever comes first. Either one raises `administrator_granted`, unless a sign-in that sees it - first fails with a server error after the role write; the retry then gains nothing. An account - that never signs in again and holds no live session raises nothing. So watch membership of the - mapped group in the directory itself. + whichever comes first. Either one raises `administrator_granted`, but at least these cases raise + none. A sign-in that sees it first and fails with a server error after the role write loses it; + the retry then gains nothing. A reconciler pass the mass-revoke breaker aborts writes no roles. + An account that never signs in again raises nothing when no pass reaches it: it holds no live + session, or `[auth].ad_session_recheck_seconds` is 0. So watch membership of the mapped group in + the directory itself. **CORRECTED 2026-10-01:** this read "raises no alert. The engine never sees that grant." **CORRECTED 2026-10-06:** this read "raises no `administrator_granted` alert", and the sign-in case below read "It raises no alert." Vault BACKLOG #2610 added the sign-in alert. @@ -4118,8 +4120,8 @@ writes no roles, and it strikes and revokes as an absent account does, with the 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, and one whose every lookup was referred, still strikes the row, because its answer never came from -the directory (BACKLOG #3170). Neither pass revokes it: that waits for the next pass that is not -aborted. A referral's `auth.ad_reconcile_referred` row and alert count only +the directory (BACKLOG #3170). Neither pass revokes it: only a pass that is not aborted can, once +the row's strikes reach the threshold and the row is in that pass's probe budget. A referral's `auth.ad_reconcile_referred` row and alert count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since BACKLOG #1143 slice C sits on a row probed by its id (ADR 0184 AC-5). A binding already on an id-less @@ -4134,7 +4136,8 @@ Removing the binding (`DELETE /users/{user_id}/federated-identity`) returns the The pass has these safety properties, because the lookup still returns one indistinguishable "not found" for *deleted*, *moved out of the search base* and *the search base was never right*: -- **Fail-OPEN.** An unreachable domain controller revokes **nothing** and does not even accrue a strike. +- **Fail-OPEN.** An unreachable domain controller revokes **nothing**, and an account the pass asked + about accrues no strike. An id-less row, never asked, is still struck (BACKLOG #3170). A fail-closed re-check would turn a directory blip into a total console outage during exactly the incident when operators need the console. - **Two strikes** (`ad_session_recheck_strikes`, default 2) before any revocation. @@ -4192,7 +4195,7 @@ refers on every pass. Read as an outage, it would stop revocation on a first dep page. The two bases fail differently. A referring user search base refers every probe that searches, -so the pass judges nothing and revokes nothing, like an outage, but it still pages. A referring +so the pass judges no account it asked about and revokes nothing, like an outage, but it still pages. A referring group search base refers only the accounts the user search found enabled. Disables and deletions still revoke, but a role or channel-scope change in the directory does not reach those accounts' sessions while it diff --git a/messagefoundry/auth/reconcile.py b/messagefoundry/auth/reconcile.py index 230586d7b3..d32146136c 100644 --- a/messagefoundry/auth/reconcile.py +++ b/messagefoundry/auth/reconcile.py @@ -33,7 +33,8 @@ A referral is not an outage, and not a reason to stop the pass either (BACKLOG #2538). A :attr:`ProbeOutcome.REFERRED` account is left unjudged, like an unreachable one, and every other probe in the pass is judged as usual. Unlike an outage, a referral alerts. A pass whose only answers are -referrals and failures aborts as ``directory_referral``, which judges nothing and alerts. +referrals and failures aborts as ``directory_referral``, which judges no account it asked about +and alerts. Such a pass, like an outage, still strikes an UNKEYED row (BACKLOG #3170). The pass also re-diffs channel scope (ADR 0198, BACKLOG #1957). It revokes when the directory would withdraw or narrow a principal's scope, and it never writes the scope: the next login does, through @@ -258,7 +259,8 @@ def directory_referral(self) -> bool: def judged_nothing(self) -> bool: """True when the pass aborted before judging any account: an outage, or a pass of referrals and failures only. Such a pass leaves the hold's state, its message and every clear alone, - because it learned nothing about the accounts.""" + because it learned nothing about the accounts it asked about. It still strikes an UNKEYED + row, which was judged without the directory (BACKLOG #3170).""" return self.directory_outage or self.directory_referral @property From 71addac55377cd26607319e3eb8260b39e1579ac Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 00:08:29 -0500 Subject: [PATCH 19/27] Review round one on the b257-dchain merge: count and strike wording Applies round-one findings that concern text and tests the merge itself wrote; nothing here changes behaviour. - pipeline/alerts.py, docs/PHI.md: the ad_reconcile_aborted count for a probe fault is no longer called "judged". It is the same total as the breaker's, the candidates the pass took up, asked or not, which counts the faulted accounts left unjudged. - docs/SECURITY.md, changelog.d/3170.security.md: an id-less row is struck on any pass that aborts on referrals, probe faults or failures, not only on one whose every lookup was referred or faulted. - tests/test_ad_session_reconcile.py: the unkeyed-strike cases pin the abort slug, so a faults-only pass cannot pass as an outage. Left open, for the Manager: the probe fault's alert and its auth.ad_reconcile_faulted row count plan.probed while its latched message counts plan.asked (main's code, PR 2342); #3170's rule for the referral would apply to it unchanged. Proposed PR title: b257 D1-D3 auth follow-ups: TOTP refusal row, directory notices, revoke-miss audit, OIDC step-up follow-ups, sign-in timeout, reconcile admin grant and unkeyed counts --- changelog.d/3170.security.md | 3 ++- docs/PHI.md | 2 +- docs/SECURITY.md | 4 ++-- messagefoundry/pipeline/alerts.py | 13 ++++++----- tests/test_ad_session_reconcile.py | 36 ++++++++++++++++++------------ 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/changelog.d/3170.security.md b/changelog.d/3170.security.md index 7a0474d49c..9008375017 100644 --- a/changelog.d/3170.security.md +++ b/changelog.d/3170.security.md @@ -4,7 +4,8 @@ field in place of `probed`, and that row, the referral's latched message and its `ad_reconcile_aborted` alert count only the lookups the pass sent. The breaker's own alert still counts every account it judged. Such a row is also struck now on a pass that is a directory - outage or got only referrals or probe faults, as on any other pass. Such a pass still revokes nothing. Only a + outage or aborted on referrals or probe faults, as on any other pass. Such a pass still revokes + nothing. Only a pass that is not aborted can, once the row's strikes reach the threshold and the row is in that pass's probe budget, so while every pass aborts its sessions last to their own expiry. Its revocation still counts toward the mass-revoke breaker, as before. (`vault BACKLOG #3170`) diff --git a/docs/PHI.md b/docs/PHI.md index 4327544ea5..c44aa755b5 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1808,7 +1808,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP listener's `tls_handshake_failed` (vault BACKLOG #2613 -- a TLS handshake on a listener that runs its own handshakes (the standard asyncio loop, not uvloop) failed; the first at once, then one per listener per minute carrying the count since the last, and the reason is a count and an exception class name, never the peer's bytes), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100; the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker or the probe fault judged, or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under the breaker's or the referral's source but never the probe fault's, which has no inverse, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100; the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker judged; for a probe fault the same total of candidates the pass took up, asked or not, faulted ones included; or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` for the same file on the same engine process, once the reload's record no longer holds the refusal, and when the reload stops -- vault BACKLOG #2904), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_aborted`, under the breaker's or the referral's source but never the probe fault's, which has no inverse, and `ad_reconcile_held`; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-two-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 3dfb569571..cc5008b5ea 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -4143,8 +4143,8 @@ writes no roles, and it strikes and revokes as an absent account does, with the `directory_object_id_missing`. It is not a `userAccountControl` answer, so it never moves the ADR 0195 hold. It is never asked, so it is no part of an outage either: a pass whose every lookup fails still audits `auth.ad_reconcile_skipped`, and that row's `asked` count leaves it out. Such a pass, -and one whose every lookup was referred or faulted, still strikes the row, because its answer never came from -the directory (BACKLOG #3170). Neither pass revokes it: only a pass that is not aborted can, once +and one that aborts because its lookups were only referred, faulted or failed, still strikes the +row, because its answer never came from the directory (BACKLOG #3170). No such pass revokes it: only a pass that is not aborted can, once the row's strikes reach the threshold and the row is in that pass's probe budget. A referral's `auth.ad_reconcile_referred` row and alert count only the lookups sent too. Its revocation does count toward the mass-revoke breaker below. Such a row cannot take a federated binding: the bind refuses it, so every binding the bind has made since diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index ffa60a9db5..8db451c29d 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -546,10 +546,11 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s ``name`` labels the source: ``"directory-reconciler"`` for the breaker, ``"directory-reconciler-referral"`` for a referral, and ``"directory-reconciler-probe-fault"`` for a probe fault, so each is its own instance and - throttle. ``reason`` is one of those three closed-set slugs. ``probed`` is, for the breaker - and for a probe fault, how many principals the pass judged, the breaker's denominator, and - for a referral how many probes the pass sent to the directory, which leaves out an id-less - row (BACKLOG #3170). ``detail`` is the operator-facing explanation the auth service + throttle. ``reason`` is one of those three closed-set slugs. ``probed`` is, for the breaker, + how many principals the pass judged, its denominator; for a probe fault, the same total of + candidates the pass took up, asked or not, which counts the faulted accounts it left + unjudged and any id-less row; and for a referral how many probes the pass sent to the + directory, which leaves out an id-less row (BACKLOG #3170). ``detail`` is the operator-facing explanation the auth service latches. No PHI. Emitted by the API-lifespan reconciler task, never from ``auth/``. A whole-directory outage is NOT this event: it is audited as ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine.""" @@ -1093,8 +1094,8 @@ def no_enabled_administrator(self, name: str, *, via: str) -> None: def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: str) -> None: log.warning( - # The count is the judged total for the breaker and a probe fault, or a referral's probes - # asked (BACKLOG #3170). + # The count is the candidates the pass took up for the breaker and a probe fault, or a + # referral's probes asked (BACKLOG #3170); see AlertSink.ad_reconcile_aborted. "ALERT ad_reconcile_aborted: %r left accounts unrevoked, counting %d principal(s) " "(%s): %s", name, diff --git a/tests/test_ad_session_reconcile.py b/tests/test_ad_session_reconcile.py index 394d85806c..3fdac67123 100644 --- a/tests/test_ad_session_reconcile.py +++ b/tests/test_ad_session_reconcile.py @@ -664,20 +664,26 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N @pytest.mark.parametrize( - "asked", + ("asked", "aborted"), [ - [reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE)], - [reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED)], - [ - reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), - reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED), - ], + ([reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE)], "directory_unavailable"), + ([reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED)], reconcile.REFERRAL_ABORT), + ( + [ + reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), + reconcile.Probe("r", "r", reconcile.ProbeOutcome.REFERRED), + ], + reconcile.REFERRAL_ABORT, + ), # BACKLOG #3100 shares the referral's early return, so a pass of faults only is one too. - [reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT)], - [ - reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), - reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT), - ], + ([reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT)], reconcile.PROBE_FAULT_ABORT), + ( + [ + reconcile.Probe("a", "a", reconcile.ProbeOutcome.UNAVAILABLE), + reconcile.Probe("f", "f", reconcile.ProbeOutcome.FAULT), + ], + reconcile.PROBE_FAULT_ABORT, + ), ], ids=[ "outage", @@ -688,15 +694,17 @@ def test_an_unkeyed_probe_does_not_hide_an_outage_or_a_referral_only_pass() -> N ], ) def test_an_unkeyed_row_is_struck_on_a_pass_that_judged_nothing( - asked: list[reconcile.Probe], + asked: list[reconcile.Probe], aborted: str ) -> None: """BACKLOG #3170 item 3. An UNKEYED row is judged without the directory, so a pass that learned nothing from it still strikes the row. Only the strike: the asked probes' strikes, the outcome - record the hold reads, and the hold's latch all stay as they were. + record the hold reads, and the hold's latch all stay as they were. The abort slug is pinned + too, so an id-less row cannot hide which kind of pass this was (BACKLOG #2434, #3100). RED before #3170: both early returns carried no strikes, so such a row went unstruck.""" unkeyed = reconcile.Probe("k", "k", reconcile.ProbeOutcome.UNKEYED) plan = _plan([*asked, unkeyed], prior_strikes={"k": 1, "a": 1, "r": 1, "f": 1}, latched=True) + assert plan.aborted == aborted assert plan.judged_nothing assert plan.strikes == {"k": 2} assert plan.outcomes == {} and plan.latched and not plan.hold From 70f6c45f8af48f2c2529b1d62b831401199d5289 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 12:03:44 -0500 Subject: [PATCH 20/27] Bind the step-up floor marker into the refuse partial once Review of the main merge into b257-dchain: the resolution spelled extra=floor_marker on each of the 13 federated step-up refusals past the floor, so a new arm written as refuse(REASON) would silently drop callback_floor_exempt_amr. The marker is now bound into the refuse partial once, right after it is computed, as the address fields are bound above (vault BACKLOG #3145). No arm past that point passes its own extra, and the audit rows are unchanged. BACKLOG #3220. --- messagefoundry/auth/service.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 8056d58457..d6b4114add 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -5111,6 +5111,8 @@ async def complete_oidc_step_up( # BACKLOG #3220: every refusal past the floor carries the marker when the exemption applied, # so a failed exempted step-up is as visible to the loosening's review as a served one. floor_marker = _floor_exempt_marker(floor_exempt_amr) + # Bound once, as the address fields are above, so a new arm cannot leave the marker out. + refuse = partial(refuse, extra=floor_marker) if arrived_too_early and not floor_exempt_amr: # The opted-in site's too-early callback whose amr names no exempt value (BACKLOG #2388). return await refuse(TOO_EARLY) @@ -5125,10 +5127,10 @@ async def complete_oidc_step_up( # stepped up, so the operator is not told "verified" and then signed out. or not session.is_live(now=now, idle_seconds=self.session_idle_seconds) ): - return await refuse("session_gone", lost=True, extra=floor_marker) + return await refuse("session_gone", lost=True) user = await self._load_user(session.user_id) if user is None or user.disabled or user.auth_provider != AuthProvider.AD.value: - return await refuse("session_gone", lost=True, extra=floor_marker) + return await refuse("session_gone", lost=True) if (user.oidc_issuer, user.oidc_subject) != ( principal_claims.issuer, principal_claims.subject, @@ -5141,14 +5143,14 @@ async def complete_oidc_step_up( # BEFORE both freshness tests (BACKLOG #2143), so another person's answer that passed # the claims ladder is filed as a subject mismatch whatever its auth_time. Both tests # below judge this session's identity, so neither is the right name for someone else. - return await refuse(STEP_UP_SUBJECT_MISMATCH, extra=floor_marker) + return await refuse(STEP_UP_SUBJECT_MISMATCH) # FRESHNESS, two tests, and both must pass. max_age=0 and prompt=login ask the IdP to # authenticate the user afresh, so a conforming IdP's auth_time postdates this request. # (a) Against our clock: auth_time is IdP clock and issued_at is ours, so the floor allows # the configured skew for an IdP clock that runs behind. skew = self._settings.oidc_clock_skew_seconds if flow.issued_at <= 0 or principal_claims.auth_time < flow.issued_at - skew: - return await refuse(STEP_UP_NOT_FRESH, extra=floor_marker) + return await refuse(STEP_UP_NOT_FRESH) # (b) Against the IdP's own clock (BACKLOG #2143): auth_time must be LATER than the one the # session holds, which is the sign-in's or the last step-up's. No skew applies, because # both values come from the IdP. This closes most of what (a) alone left: an IdP that @@ -5162,20 +5164,20 @@ async def complete_oidc_step_up( # row and the operator's text say which one refused. held_auth_time = session.idp_auth_time if held_auth_time is None: - return await refuse(STEP_UP_IDP_AUTH_TIME_MISSING, extra=floor_marker) + return await refuse(STEP_UP_IDP_AUTH_TIME_MISSING) if principal_claims.auth_time <= held_auth_time: - return await refuse(STEP_UP_IDP_AUTH_TIME_NOT_LATER, extra=floor_marker) + return await refuse(STEP_UP_IDP_AUTH_TIME_NOT_LATER) # The DIRECTORY still has the account. The password re-bind this leg replaces failed for a # disabled or deleted AD object, and the sign-in leg refuses one as not_in_directory. An IdP # can still sign such a user in (sync lag, or its own credential store), so the engine asks # the directory itself, by the row's immutable id as the sign-in leg does (BACKLOG #1532). if self._ldap is None: - return await refuse("not_configured", extra=floor_marker) + return await refuse("not_configured") object_id = _directory_key(user) if object_id is None: # Never a name-only lookup: a directory may reissue a freed name to someone else. The # sign-in leg refuses such a row for the same reason (BACKLOG #2027). - return await refuse(DIRECTORY_OBJECT_ID_MISSING, extra=floor_marker) + return await refuse(DIRECTORY_OBJECT_ID_MISSING) try: principal = await asyncio.to_thread( self._ldap.resolve_principal, user.username, object_id=object_id @@ -5184,14 +5186,14 @@ async def complete_oidc_step_up( _log.warning( "directory lookup failed during a federated step-up: %s", type(exc).__name__ ) - return await refuse("directory_unavailable", extra=floor_marker) + return await refuse("directory_unavailable") if principal is None: - return await refuse("not_in_directory", extra=floor_marker) + return await refuse("not_in_directory") # BACKLOG #3174: the answer must name the row's own directory object before its groups are # trusted, by the test the sign-in leg and the AD step-up re-bind apply. mismatch = _directory_answer_mismatch(principal, object_id) if mismatch is not None: - return await refuse(mismatch, extra=floor_marker) + return await refuse(mismatch) # BACKLOG #2154: the account's stored roles must all be among the roles its current groups # map to, the same test _directory_step_up_refusal applies on the TOTP and passkey legs # (#2240). Inline rather than through that helper, because its probe would change this @@ -5200,9 +5202,9 @@ async def complete_oidc_step_up( # as before, through the loader (BACKLOG #2592). fresh = await self._load_user(user.id) if fresh is None: - return await refuse("session_gone", lost=True, extra=floor_marker) + return await refuse("session_gone", lost=True) if not set(fresh.role_ids) <= await self._ad_roles_for_groups(principal.groups): - return await refuse(DIRECTORY_ROLES_DEMOTED, extra=floor_marker) + return await refuse(DIRECTORY_ROLES_DEMOTED) purpose = flow.step_up_purpose # The password leg's three ORDER-CRITICAL steps (see :meth:`reauth`), against the hash. # (1) Every stamp against the OLD hash, re-anchoring the session to this client address. From 9fd48a0718409a2033c6024389cc1ad2227e6e45 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 15:20:03 -0500 Subject: [PATCH 21/27] A self-revoke that loses the rotation race writes the miss row Review of the fourth main merge. Main's #3029 added a second 404 path to revoke_own_session: the revoke matched no unrevoked row, because a rotation re-keyed the session or a concurrent revoke ended it first. That path returned False with no audit row, while this branch's vault #3260 limb 2 (owner ruling 2026-10-16) has every miss of DELETE /me/sessions/{id} write one auth.session_revoke_missed row, as docs/SECURITY.md states. The race miss now writes that same row, and still no auth.session_revoked row. - service.py: the supersede_session miss joins the unknown and foreign id branch; docstring says so. - tests/test_session_supersession.py: the race test also pins the miss row. - changelog.d/3029.security.md: no longer says the 404 writes no row. - _ui_seam.py: a paragraph names the #3029 logout return type in the digest. BACKLOG #3029, vault BACKLOG #3260. --- changelog.d/3029.security.md | 3 ++- messagefoundry/api/_ui_seam.py | 5 +++++ messagefoundry/auth/service.py | 17 ++++++++++------- tests/test_session_supersession.py | 6 ++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/changelog.d/3029.security.md b/changelog.d/3029.security.md index 5c17c92e4c..27dc3f97e3 100644 --- a/changelog.d/3029.security.md +++ b/changelog.d/3029.security.md @@ -7,5 +7,6 @@ `auth.logout` rows carry `detail.outcome`, `revoked` or `nothing_to_revoke`, and a console sign-out's row names the owner of the session it ended instead of a NULL actor. Both sign-out routes still answer the same either way. A self-revoke that - ends nothing answers 404 and writes no row. A sign-out or self-revoke that loses that race still + ends nothing answers 404 and writes no `auth.session_revoked` row, only the + `auth.session_revoke_missed` row an unknown id writes (vault BACKLOG #3260). A sign-out or self-revoke that loses that race still leaves the session live under its new token; ending it there is not attempted. (`BACKLOG #3029`) diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index ec1b395606..02549e987a 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -416,6 +416,11 @@ #: none of which moved it. The value below covers all four, and vault BACKLOG #3260 above: it was #: re-derived again on the batch 257 branch's merge of those changes. #: +#: BACKLOG #3029: `AuthService.logout` returns a `LogoutOutcome` where it returned `None`. The +#: console's `/ui/logout` ignores the value, but the return type is part of the discovered +#: signature, so the digest moved on main with it. The value below covers it too: it was +#: re-derived on the batch 257 branch's fourth merge of main. +#: #: The digest below covers the surface DISCOVERED from the console's own imports and uses, which is #: strictly larger than the five hand-maintained tuples it replaced -- those had drifted, and the #: proof is that commit 40a4d5d9 added a REQUIRED ``UploadedFileList.scope`` field the console renders diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 70e397f9e2..7045175715 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -8971,12 +8971,17 @@ async def revoke_own_session( the answer the caller is owed. ``client`` is required, as on :meth:`audit_self_target_refused`, because both planes call this from a request. - Also ``False``, with no audit row, when the revoke matched no unrevoked row (BACKLOG #3029): - at least a rotation that re-keyed the session after the read, which leaves it live under - its new id, or a concurrent revoke that ended it first. The revoke reports whether it - matched, so neither the return nor the trail claims a revoke that did not happen.""" + Also ``False`` when the revoke matched no unrevoked row (BACKLOG #3029): at least a + rotation that re-keyed the session after the read, which leaves it live under its new id, + or a concurrent revoke that ended it first. The revoke reports whether it matched, so + neither the return nor the trail claims a revoke that did not happen. That miss answers the + same 404, so it writes the same miss row and no ``auth.session_revoked`` row.""" session = await self._load_session(session_id) - if session is None or session.user_id != identity.user_id: + if ( + session is None + or session.user_id != identity.user_id + or await self._store.supersede_session(session_id, now=time.time()) is None + ): # The console passes the id as plain text, so only an id-shaped one is copied into the # trail; anything else is recorded as malformed, never as the caller's own characters. shown = session_id[:12] if _SESSION_ID_SHAPE.fullmatch(session_id) else "malformed" @@ -8993,8 +8998,6 @@ async def revoke_own_session( defects=(), ) return False - if await self._store.supersede_session(session_id, now=time.time()) is None: - return False await self._audit( "auth.session_revoked", actor=actor, diff --git a/tests/test_session_supersession.py b/tests/test_session_supersession.py index 02572139b5..6b8b01b43f 100644 --- a/tests/test_session_supersession.py +++ b/tests/test_session_supersession.py @@ -371,6 +371,12 @@ async def test_a_rotation_racing_a_self_revoke_is_not_reported_as_a_revoke( assert rotated is not None and await _live(service, rotated), "the race was not driven" assert revoked is False, "revoke_own_session reported a revoke that did not happen" assert await _superseded(store, "self") == [], "an audit row recorded a revoke that missed" + # Vault BACKLOG #3260, limb 2: the miss answers the same 404 as an unknown id, so it writes the + # same miss row, naming the id the caller sent. + missed = await store.list_audit(action="auth.session_revoke_missed") + assert [(row["actor"], json.loads(str(row["detail"]))) for row in missed] == [ + ("op", {"session": target_hash[:12]}) + ], "the race's 404 left no miss row" async def test_a_self_revoke_that_lands_returns_true_and_audits_it(store: MessageStore) -> None: From 8d4d1138ec3fa9b481a158216bb8f82ca2802941 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 23:15:11 -0500 Subject: [PATCH 22/27] The reconcile grant race test's spy takes the watermark and returns the outcome Main's 838a5626d (BACKLOG #3085) changed AuthService._apply_reconcile_revocation from (revocation) -> bool to (revocation, *, watermark) -> "skipped" | "quiet" | "reported". The vault BACKLOG #3141 race test in tests/test_directory_last_admin_alert.py wrapped it with the old signature, so mypy's tests leg refused it and the call would raise TypeError. The spy now forwards the watermark and returns the real outcome, as b257-dtail's merge a4f88a42a does. --- tests/test_directory_last_admin_alert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_directory_last_admin_alert.py b/tests/test_directory_last_admin_alert.py index 0631fd0f75..d739f92124 100644 --- a/tests/test_directory_last_admin_alert.py +++ b/tests/test_directory_last_admin_alert.py @@ -399,11 +399,11 @@ async def test_the_grant_is_judged_on_the_roles_read_at_apply_time_not_the_plans rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) real = rig.service._apply_reconcile_revocation - async def a_sign_in_lands_first(revocation: Any) -> bool: + async def a_sign_in_lands_first(revocation: Any, *, watermark: Any) -> str: await rig.store.set_user_roles( revocation.user_id, [Role.ADMINISTRATOR.value], assigned_by="ad-sync" ) - return await real(revocation) + return await real(revocation, watermark=watermark) monkeypatch.setattr(rig.service, "_apply_reconcile_revocation", a_sign_in_lands_first) From 8a7fa2b9a0654e8e36eb621c9e07649978e1b736 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sat, 17 Oct 2026 23:31:29 -0500 Subject: [PATCH 23/27] The reconcile grant race test reaches the grant check again After the merge of main, the vault BACKLOG #3141 race test in tests/test_directory_last_admin_alert.py passed whatever before-set the service judged the grant on. Its spy wrote exactly the pass's target roles, so the BACKLOG #3085 apply-time check found nothing changed and skipped the role write and the grant check with it. The spy now writes Operator beside Administrator, so the pass's own write still runs and adds nothing, and the test asserts that write ran. Measured: with the grant judged on an empty before-set the test now fails; restored, it passes. The spy is also typed with SessionRevocation, _AuditWatermark and _RevocationApplied, as the sibling spies in test_ad_session_reconcile.py are, so a further signature change reaches mypy's tests leg. --- tests/test_directory_last_admin_alert.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/test_directory_last_admin_alert.py b/tests/test_directory_last_admin_alert.py index d739f92124..ad4530b869 100644 --- a/tests/test_directory_last_admin_alert.py +++ b/tests/test_directory_last_admin_alert.py @@ -29,7 +29,13 @@ from messagefoundry.api.security import DIRECTORY_GRANTED_BY from messagefoundry.auth.ldap import AdPrincipal, DirectoryAnswer, DirectoryProbe from messagefoundry.auth.permissions import Role -from messagefoundry.auth.service import NO_ENABLED_ADMINISTRATOR_ACTION, AuthService +from messagefoundry.auth.reconcile import SessionRevocation +from messagefoundry.auth.service import ( + NO_ENABLED_ADMINISTRATOR_ACTION, + AuthService, + _AuditWatermark, + _RevocationApplied, +) from messagefoundry.config.settings import ( _ALERT_CONTROL_EVENT_TYPES, _ALERT_EVENT_TYPES, @@ -393,15 +399,23 @@ async def test_the_grant_is_judged_on_the_roles_read_at_apply_time_not_the_plans ) -> None: """A sign-in that lands between the pass's read and its write has already written Administrator, and paged for it. The write then adds nothing, so the pass must not page again. - RED when the before-set is taken from the plan, which read the roles before that sign-in.""" + RED when the before-set is taken from the plan, which read the roles before that sign-in. + + The sign-in writes Operator beside Administrator, which the pass's write then removes. Were the + stored roles equal to the pass's target, #3085's apply-time check would skip the write and the + grant check with it, and this test would pass whatever before-set the service used.""" rig.ldap.present["jdoe"] = _principal("jdoe", OPERATORS) await rig.sign_in("jdoe") rig.ldap.present["jdoe"] = _principal("jdoe", ADMINS) real = rig.service._apply_reconcile_revocation - async def a_sign_in_lands_first(revocation: Any, *, watermark: Any) -> str: + async def a_sign_in_lands_first( + revocation: SessionRevocation, *, watermark: _AuditWatermark | None + ) -> _RevocationApplied: await rig.store.set_user_roles( - revocation.user_id, [Role.ADMINISTRATOR.value], assigned_by="ad-sync" + revocation.user_id, + [Role.ADMINISTRATOR.value, Role.OPERATOR.value], + assigned_by="ad-sync", ) return await real(revocation, watermark=watermark) @@ -410,6 +424,8 @@ async def a_sign_in_lands_first(revocation: Any, *, watermark: Any) -> str: plan = await rig.service.reconcile_directory_sessions() assert [(r.username, r.reason) for r in plan.revocations] == [("jdoe", "roles_changed")] + # The control: the pass's own write ran (Operator is gone), so the grant check was reached. + assert await rig.roles("jdoe") == [Role.ADMINISTRATOR.value] assert rig.grants == [] From a4d22513212d31f1d040e3867839ab340ec44aeb Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sun, 18 Oct 2026 09:35:39 -0500 Subject: [PATCH 24/27] Merge review fixes: provision refusal says remove, doc limits stated (BACKLOG #3113 #3164 #3121 #3260 #3145 #2950 #3141 #3170) From the xhigh review of merge ee20db1ca: - provision-admin's enable refusal says "remove the authenticator entry it showed" again (main's wording). Main's #3025 compare-and-set binds each enable to its own run's staged cell, so the refused run's entry is never live; #3113 item 1's "keep it" advice rested on the closed race and, followed, would spend wrong codes against the second-step lock. Comment rewritten. - The race test is renamed, pins the remove wording, bounds both waits with asyncio.wait_for, and types its task as ProvisionedAdministrator. - SECURITY.md TOTP paragraph: the refusal row is written unless its own write fails (logged at ERROR); the re-sign-in clause no longer claims the session always ended. - ASVS-L2-PHASE0-CHANGES.md: the TOTP row no longer says the enable does not check the staged secret; #3025 made it a compare-and-set. - _audit_enroll_refused write_lost comment: the revoke is requested, and session_ended says whether it happened. --- docs/ASVS-L2-PHASE0-CHANGES.md | 2 +- docs/SECURITY.md | 6 +++-- messagefoundry/auth/service.py | 27 +++++++++------------ tests/test_provision_first_administrator.py | 21 ++++++++-------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/ASVS-L2-PHASE0-CHANGES.md b/docs/ASVS-L2-PHASE0-CHANGES.md index a560c19fe2..35c24c9920 100644 --- a/docs/ASVS-L2-PHASE0-CHANGES.md +++ b/docs/ASVS-L2-PHASE0-CHANGES.md @@ -468,7 +468,7 @@ is right. | Off-box log-forward client key | 4, on the same condition as the API TLS server key | Loads the combined PEM in `_build_tls_context` ([`logging_setup.py`](../messagefoundry/logging_setup.py)). There is no passphrase setting | As for the API TLS server key. The file's permissions are the key's only protection, so setting them is the site's job | | Native API client key | 4, on the same condition as the API TLS server key | Holds no copy. The client process loads it (`_build_verify_context` in [`apiclient/client.py`](../messagefoundry/apiclient/client.py)) | Every function in this row belongs to the operator of the client machine | | Non-production self-signed key | Two keys, each 4, private authentication key, like the API TLS placeholder key. The `cert self-signed` key is a placeholder a site may serve. The load-test harness pair is also test keying material under section 8.1.4 | Mints both with `make_self_signed` in [`pki.py`](../messagefoundry/pki.py), the CLI key on request. The CLI writes its key through `_write_private_key`, which refuses to overwrite an existing file. The harness writes its pair with a plain write and `chmod`, as the lifecycle row above says. Nothing is audited | The `cert self-signed` key is a placeholder key, like the API TLS placeholder key: a site may serve it until it supplies a chain. Set its cryptoperiod under section 5.3.6 item 4 through `--days`, because the engine never renews it. Replace it with an operator-supplied chain, then delete both files. The load-test harness pair is test keying material, which section 8.1.4 says shall not be used operationally, and the engine does not stop that | -| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable does not check that the staged secret is the one the code proved; [`SECURITY.md`](SECURITY.md), in its TOTP paragraph, states that gap (vault BACKLOG #3025; this sentence corrected under vault BACKLOG #3113). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | +| TOTP shared secret | 3, symmetric authentication key: an HMAC key shared by exactly two parties | **Pre-activation (7.1):** `begin_mfa_enrollment` in [`auth/service.py`](../messagefoundry/auth/service.py) stores a fresh staged secret and audits `auth.mfa_enroll_started`. A later call replaces it. A staged secret nobody confirms stays in the column until then, or until a disable or reset; nothing moves it on by time. **Key confirmation, then active (7.1, transition 4):** `confirm_mfa_enrollment` turns MFA on only after a live code checks out against the secret it read, and audits `auth.mfa_enrolled`. The enable is a compare-and-set on the staged secret the code proved, so a secret re-staged in between is refused rather than enabled; [`SECURITY.md`](SECURITY.md), in its TOTP paragraph, states the refusals (BACKLOG #3025; this sentence corrected under vault BACKLOG #3113, and again once #3025 merged). **Out of use:** `disable_totp` sets the live column to NULL on a disable or an administrator reset, audited as `auth.mfa_disabled` or `auth.mfa_reset`. That removes the live copy. It is not destruction to the 8.3.4 bar (point 3), because older copies can survive. **Backup:** where a DR archive carries the store, it carries the secret; the lifecycle row above says when discarding the DEK erases those copies. Table 7 allows backup of this key type, and B.3.2 (its case 3) says a key that can be re-established need not be backed up. Re-enrolment re-establishes this one. **Cryptoperiod: none.** The secret is exempt from a calendar lifetime (owner ruling 2026-09-27, BACKLOG #1931). Section 5.3.6 item 3 suggests an originator-usage period of no more than two years. So this is a recorded departure from that suggestion, not an alignment with it | Keep the user's copy in one authenticator. After an administrator reset, tell the user to delete the old entry. Section 8.3.5 says the party revoking a key shared by two shall inform the other | | Anonymizer re-identification salt | None of the 19 types in 5.1.1 fits. It keys a hash that pseudonymizes one dataset. The nearest fit is 5.1.2's *other secret information*, so this row borrows 5.3.7's rule for that and takes no cryptoperiod from 5.3.6 | Never mints or stores it. `Keyer` in [`anon/keying.py`](../messagefoundry/anon/keying.py) refuses a visibly weak salt, and the export reads it from the environment for one run | Mint it with a CSPRNG, one per dataset, and hold it with one person. Section 5.3.7 says other secret information shall not be kept longer than necessary, so its period is one dataset. Keep no backup unless the dataset must be regenerated, and destroy it when that need ends | | Sealed-cache key | 6, symmetric data-encryption key. It protects the confidentiality and integrity of decoded cache values in the engine's own memory | Mints it per process (`_Sealer` in [`store/sealed_cache.py`](../messagefoundry/store/sealed_cache.py)) and uses it only to seal and open cache values, with each value bound to its cache key as associated data. Pre-activation and active are one step, and the key is destroyed with the process, so its cryptoperiod (5.3.6) is the process lifetime. There is no backup or archive of it (Table 7 is moot) | Protect the engine process's memory: the in-use residual in `docs/PHI.md` §3 and the host requirements in §10 apply, because this key sits in the same memory as the values it seals | diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 39470158c7..5f3efd3049 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -3056,12 +3056,14 @@ confirm asks the store to end the session it just rotated, and answers as a lost `secret_cleared`, `secret_replaced` and `write_lost`. Each is read from the row after the refusal, so a `reason` points an operator at the likely cause rather than proving it. The row's `session_ended` is false when the store refused to end the rotated session. The token is still not -handed back, and the row is written whatever the revoke raised (vault BACKLOG #3113). The compare is +handed back, and the row is written whatever the revoke raised, unless its own write fails, +which is logged at ERROR (vault BACKLOG #3113). The compare is on the stored ciphertext, so at least a store-key rotation that re-encrypts the column inside the window refuses too. That refusal reads as `secret_replaced` although no second enrolment started. `write_lost` means the row still held the verified secret with TOTP off, so the enable lost to some other concurrent write. After a `write_lost` or a store-key rotation the staged secret still -stands. The user signs in again, because the refused confirm ended the session, and passes the +stands. The user signs in again, because the refused confirm hands no token back and asks the +store to end the session, and passes the step-up `POST /me/mfa/confirm` asks of any confirm: a password proof bound to the confirm action (ADR 0077), or a recent step-up window where `[auth].require_action_step_up` is off. The user then confirms with a code from a later step, because the refused confirm spent diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index a421f726f4..6368a40f66 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -3559,23 +3559,19 @@ async def provision_first_administrator( # another run that enabled TOTP first, or another run that staged its own secret, since # the secret was staged above makes it match no row. Refused before the role, so the # row stays roleless and a re-run completes it. - # - # The words below say to keep the entry this run showed (vault BACKLOG #3113). They - # were written while staging was unconditional, when another run's enable could turn - # on THIS run's secret. The #3025 compare-and-set binds each enable to the cell its - # own run staged, so that case no longer arises; the cautious words are kept rather - # than changed in a merge. + # Each enable is bound to the cell its own run staged, so no other run can turn on + # THIS run's secret, and the entry it showed is never the live one: the words say to + # remove it. (Vault BACKLOG #3113 item 1 had them say keep it, for the re-stage race + # the #3025 compare-and-set closed.) if not await self._store.enable_totp( user_id, recovery_code_hashes=hashes, expected_secret_cell=staged_cell ): raise FirstAdministratorRefused( "the authenticator enrolment was changed while this command ran, so this run " - "did not turn TOTP on and granted no role. Another run may have turned on the " - "secret this run showed, so keep the authenticator entry it showed until a " - "sign-in to this account shows which entry works. " - "If another provision-admin run is in progress, let it finish. Then run the " - "command again: it reads that account and either completes it or says why it " - "cannot" + "did not turn TOTP on and granted no role; remove the authenticator entry it " + "showed. If another provision-admin run is in progress, let it finish. Then " + "run the command again: it reads that account and either completes it or says " + "why it cannot" ) # BACKLOG #2019: a repair can take over an account somebody else holds -- one an administrator # created with no roles, say -- so its earlier holder is told, at the address they held. @@ -11778,9 +11774,10 @@ async def _audit_enroll_refused( reason = "secret_replaced" else: # The row still holds the verified cell with TOTP off, so the write lost to some other - # concurrent write to the row (BACKLOG #3025). The session is revoked by the caller, so - # a plain retry is a lost session; recovery is as for a session lost before the - # rotation (the ROTATE BEFORE ENABLING comment in ``confirm_mfa_enrollment``). + # concurrent write to the row (BACKLOG #3025). The caller asked the store to revoke the + # session first (``session_ended`` says whether it did) and hands no token back, so a + # plain retry is a lost session; recovery is as for a session lost before the rotation + # (the ROTATE BEFORE ENABLING comment in ``confirm_mfa_enrollment``). reason = "write_lost" await self._audit( "auth.mfa_enroll_refused", diff --git a/tests/test_provision_first_administrator.py b/tests/test_provision_first_administrator.py index 4217ea5f20..a3eacb6caa 100644 --- a/tests/test_provision_first_administrator.py +++ b/tests/test_provision_first_administrator.py @@ -36,6 +36,7 @@ HOLDER_NOTICE_NO_PRIOR_ADDRESS, AuthService, FirstAdministratorRefused, + ProvisionedAdministrator, ) from messagefoundry.config.secretprovider import SecretProviderError from messagefoundry.config.settings import AlertsSettings, AuthSettings @@ -2869,15 +2870,16 @@ def test_the_last_wrong_code_is_not_told_to_try_again( _OTHER_TOTP_SECRET = "ABABABABABABABAB" -async def test_the_run_that_loses_an_enrolment_race_is_not_told_to_remove_a_live_entry( +async def test_the_run_that_loses_an_enrolment_race_is_told_to_remove_its_dead_entry( monkeypatch: pytest.MonkeyPatch, ) -> None: """Two runs repair one roleless row. The second stages its secret after the first, and both reach their enables in a fixed order, the first run's first. That is the re-stage race of vault BACKLOG #3025. Since #3025 each enable is bound to the cell its own run staged, so the first run's enable is refused and the second run turns on its OWN secret, the one it showed. - The refused run's words are still the cautious ones vault BACKLOG #3113 item 1 asked for: - they do not tell its operator to remove the entry it showed.""" + So the refused run's entry is dead, and its words tell its operator to remove it (vault + BACKLOG #3113 item 1 had them say keep it, for the race #3025 closed). The waits are bounded, + so a run that refuses before its enable fails the test rather than hanging it.""" store = await MessageStore.open(":memory:") try: service = AuthService(store, AuthSettings()) @@ -2889,9 +2891,9 @@ async def test_the_run_that_loses_an_enrolment_race_is_not_told_to_remove_a_live second_at_enable = asyncio.Event() first_enabled = asyncio.Event() run: ContextVar[str] = ContextVar("run", default="first") - tasks: list[asyncio.Task[object]] = [] + tasks: list[asyncio.Task[ProvisionedAdministrator]] = [] - async def run_second() -> object: + async def run_second() -> ProvisionedAdministrator: run.set("second") return await service.provision_first_administrator( username="site-admin", password=_PASSWORD, actor="second", **second @@ -2908,7 +2910,7 @@ async def racing_enable( # The first run has staged its secret. The second run now stages its own over it, # and stops at its own enable. tasks.append(asyncio.create_task(run_second())) - await second_at_enable.wait() + await asyncio.wait_for(second_at_enable.wait(), timeout=10) try: return await real_enable( user_id, @@ -2919,7 +2921,7 @@ async def racing_enable( finally: first_enabled.set() second_at_enable.set() - await first_enabled.wait() + await asyncio.wait_for(first_enabled.wait(), timeout=10) return await real_enable( user_id, recovery_code_hashes=recovery_code_hashes, @@ -2933,15 +2935,14 @@ async def racing_enable( username="site-admin", password=_PASSWORD, actor="first", **first ) won = await tasks[0] - assert getattr(won, "repaired", False) + assert won.repaired # The #3025 invariant, measured: the secret turned on is the one the winning run showed. row = await store.get_user("roleless") assert row is not None and row.totp_enabled assert await store.get_totp_secret("roleless") == _OTHER_TOTP_SECRET message = str(lost.value) assert "did not turn TOTP on and granted no role" in message - assert "remove" not in message.lower() - assert "keep the authenticator entry it showed" in message + assert "remove the authenticator entry it showed" in message finally: # A failure above must not leave the second run running against a closed store. for task in tasks: From 2ec919740967b6aa2c8a55ca81e9802509868b60 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sun, 18 Oct 2026 18:00:29 -0500 Subject: [PATCH 25/27] Merge review fix: pin the unsealed-map step-up row, correct the seam note - tests/test_oidc_step_up.py: the #3082 unsealed role-map refusal on the IdP leg now asserts its auth.reauth row, including start_client and client_moved, so the merge of main's reason with the bound refuse partial is pinned (red when the arm calls _step_up_refused without flow=, checked by mutation). - api/_ui_seam.py: the merge note named PR 2359 as a main-side seam move; it was #2954 (PR 2377). PR 2359 moved no seam field. --- messagefoundry/api/_ui_seam.py | 5 +++-- tests/test_oidc_step_up.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index 5d8e07f0c0..3734f8c17c 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -588,8 +588,9 @@ #: #2954's field was this branch's only seam change and main already carries it. #: #: The batch 257 branch (vault BACKLOG #3260 and its siblings, at 9dffd8fa7f1c4ba2) merged with -#: main again after BACKLOG #3082 (PR 2373) and PR 2359 landed there (main at 501bfdcf5, seam -#: digest 5b6fd34b36a52c84). Neither parent's value covered both, so the move comes from the merge. +#: main again after BACKLOG #3082 (PR 2373) and #2954 (PR 2377) moved main's seam (main at +#: 501bfdcf5, seam digest 5b6fd34b36a52c84; PR 2359 moved no seam field). Neither parent's value +#: covered both, so the move comes from the merge. #: Re-derived with ``--write`` on the merged tree, so the value below covers every entry above. #: #: The digest below covers the surface DISCOVERED from the console's own imports and uses, which is diff --git a/tests/test_oidc_step_up.py b/tests/test_oidc_step_up.py index b5ef0eaaa8..03b8975d86 100644 --- a/tests/test_oidc_step_up.py +++ b/tests/test_oidc_step_up.py @@ -895,6 +895,20 @@ async def test_a_role_map_that_fails_its_seal_is_not_called_a_demotion( assert not out.ok and out.reason == DIRECTORY_ROLE_MAP_UNSEALED assert out.error and "sign out" not in out.error.lower(), out.error assert "administrator" in out.error, out.error + # The row comes from the bound refuse partial, so it carries the flow's address fields as + # the demotion arm's row does (vault BACKLOG #3145 item 4). Pins the merge of main's #3082 + # reason with that partial. + rows = [json.loads(str(r["detail"])) for r in await _audit_rows(store, "auth.reauth")] + assert rows == [ + { + "ok": False, + "provider": "ad", + "mech": "oidc", + "reason": DIRECTORY_ROLE_MAP_UNSEALED, + "start_client": "10.0.0.9", + "client_moved": False, + } + ], rows await _assert_untouched(service, token) finally: await store.close() From cff00e191691c79563b8cc40e80f2fcafbf8875a Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Sun, 18 Oct 2026 18:32:21 -0500 Subject: [PATCH 26/27] Merge review fix 2: pin the floor marker on both role arms, rename, comments - tests/test_oidc_callback_floor_exempt_amr.py: the #3220 marker test now covers DIRECTORY_ROLES_DEMOTED and DIRECTORY_ROLE_MAP_UNSEALED, so the merged role arm is pinned to the bound refuse partial that carries the marker (red, checked by mutation, when the arm calls _step_up_refused with flow= and no extra=). - auth/service.py complete_oidc_step_up: the role-test result is named refusal, not lost, which the same partial takes as its session_lost keyword; the comment above it names the unsealed-map refusal (#3082). - api/_ui_seam.py: main's 40dde1530423fece entry says "at least" and names the batch 257 (#3260) entries it does not cover. --- messagefoundry/api/_ui_seam.py | 3 ++- messagefoundry/auth/service.py | 14 +++++----- tests/test_oidc_callback_floor_exempt_amr.py | 28 +++++++++++++++++--- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/messagefoundry/api/_ui_seam.py b/messagefoundry/api/_ui_seam.py index 3734f8c17c..33e82a8eec 100644 --- a/messagefoundry/api/_ui_seam.py +++ b/messagefoundry/api/_ui_seam.py @@ -560,7 +560,8 @@ #: Vault BACKLOG #2954 (and #2956, which builds on it) merged with main after the #3189 / #3031 #: entries above landed there (main at 440d71aaf, seam digest da529979e55cb966). #2956 moves no #: seam field itself. Re-derived with ``--write`` on that merged tree to 40dde1530423fece, which -#: covered the entries above it at the time, not the #3316 or #3082 entries above it now. +#: covered the entries above it at the time. It does not cover at least the #3316, #3082 and +#: batch 257 (#3260) entries above it now. #: #: Vault BACKLOG #2954 / #2956 merged with main again after BACKLOG #3316 landed there (main at #: fc0d05e1b, seam digest 8b2fa06dd4c4938b; the branch at 40dde1530423fece). The #3316 entry diff --git a/messagefoundry/auth/service.py b/messagefoundry/auth/service.py index 0655f5965f..64e5f0d3d3 100644 --- a/messagefoundry/auth/service.py +++ b/messagefoundry/auth/service.py @@ -5697,15 +5697,17 @@ async def complete_oidc_step_up( # BACKLOG #2154: the account's stored roles must all be among the roles its current groups # map to, the same test _directory_step_up_refusal applies on the TOTP and passkey legs # (#2240), through the one shared comparison. Not through that helper, because its probe - # would change this leg's not_in_directory reason. Only a LOST role refuses; refusing - # writes nothing, and the reconciler or the next sign-in re-syncs the roles. Read after the - # directory round trip, as before, through the loader (BACKLOG #2592). + # would change this leg's not_in_directory reason. Only a LOST role refuses, or a role map + # that fails its seal, as DIRECTORY_ROLE_MAP_UNSEALED rather than a demotion (BACKLOG + # #3082). Refusing writes nothing; for a lost role the reconciler or the next sign-in + # re-syncs the roles. Read after the directory round trip, as before, through the loader + # (BACKLOG #2592). fresh = await self._load_user(user.id) if fresh is None: return await refuse("session_gone", lost=True) - lost = await self._directory_roles_lost(fresh.role_ids, principal.groups) - if lost is not None: - return await refuse(lost) + refusal = await self._directory_roles_lost(fresh.role_ids, principal.groups) + if refusal is not None: + return await refuse(refusal) purpose = flow.step_up_purpose # The password leg's three ORDER-CRITICAL steps (see :meth:`reauth`), against the hash. # (1) Every stamp against the OLD hash, re-anchoring the session to this client address. diff --git a/tests/test_oidc_callback_floor_exempt_amr.py b/tests/test_oidc_callback_floor_exempt_amr.py index 47e050287c..7945c6b609 100644 --- a/tests/test_oidc_callback_floor_exempt_amr.py +++ b/tests/test_oidc_callback_floor_exempt_amr.py @@ -14,17 +14,24 @@ import json import time import urllib.parse +from dataclasses import replace from typing import Any import pytest from cryptography.hazmat.primitives.asymmetric import rsa from messagefoundry.auth import oidc +from messagefoundry.auth import service as service_module from messagefoundry.auth.audit_visibility import ( DIRECTORY_LOCKED_REFUSAL_DETAIL, HIDDEN_FROM_READERS_WITHOUT_USERS_MANAGE, ) -from messagefoundry.auth.service import TOO_EARLY, AuthService +from messagefoundry.auth.service import ( + DIRECTORY_ROLE_MAP_UNSEALED, + DIRECTORY_ROLES_DEMOTED, + TOO_EARLY, + AuthService, +) from messagefoundry.config.settings import AuthSettings, _auth_limit_loosenings from messagefoundry.store.store import MessageStore from tests.test_auth_oidc_service import ( @@ -410,7 +417,15 @@ async def test_a_step_up_past_the_floor_records_no_exemption( await store.close() -@pytest.mark.parametrize("refusal", ["not_in_directory", "step_up_subject_mismatch"]) +@pytest.mark.parametrize( + "refusal", + [ + "not_in_directory", + "step_up_subject_mismatch", + DIRECTORY_ROLES_DEMOTED, + DIRECTORY_ROLE_MAP_UNSEALED, + ], +) @pytest.mark.parametrize(("elapsed", "marked"), [(FLOOR - 0.001, True), (FLOOR + 0.001, False)]) async def test_a_refused_step_up_carries_the_marker_only_when_the_exemption_applied( rsa_key: rsa.RSAPrivateKey, @@ -420,7 +435,8 @@ async def test_a_refused_step_up_carries_the_marker_only_when_the_exemption_appl marked: bool, ) -> None: """BACKLOG #3220, the step-up leg: a refusal after the exchange carries the marker when the - exemption let the too-early callback past the floor, and only then.""" + exemption let the too-early callback past the floor, and only then. The two role arms ride the + bound refuse partial that carries the marker since main's #3082 reason merged into it.""" store = await MessageStore.open(":memory:") try: service, clock, token = await _signed_in( @@ -429,6 +445,12 @@ async def test_a_refused_step_up_carries_the_marker_only_when_the_exemption_appl claims: dict[str, Any] = {"amr": ["mfa", EXEMPT, "x-token-only"]} if refusal == "not_in_directory": service._ldap._principal = None # type: ignore[union-attr] + elif refusal == DIRECTORY_ROLES_DEMOTED: + # The account signed in as operator; its groups now map to no role. + ldap: Any = service._ldap + ldap._principal = replace(ldap._principal, groups=frozenset()) + elif refusal == DIRECTORY_ROLE_MAP_UNSEALED: + monkeypatch.setattr(service_module, "ad_map_seal_ok", lambda _sealer, _current: False) else: claims["sub"] = "S-1-5-21-someone-else" flow_id, _url = await _begin(service, token) From 573fbd37b93de3e4f61f58815062c790e7525624 Mon Sep 17 00:00:00 2001 From: Scott Hall Date: Mon, 19 Oct 2026 00:08:48 -0500 Subject: [PATCH 27/27] Merge review fix: name the paging refusal's count, detail and inverse Review round one of the PR 2431 merge. The #3170 count text was written before per-account paging alerts existed, and now covers them: - AlertSink.ad_reconcile_aborted: a paging refusal's count is the pass's, though it concerns one account, and its detail is a fixed sentence, not a latched message. The probe-fault clause again says the faulted accounts, not the paging refusal's. - LoggingAlertSink: the line says the count is the pass's ("in a pass counting"), so one paged account no longer reads as N unrevoked. - docs/PHI.md stream 8: the paging clause's undefined "the probed count" is now "the same count as a probe fault's". - api/app.py _alert_reconcile_plan: a paging refusal has no inverse either, as PHI.md and _alert_paging_refusals already say. --- docs/PHI.md | 2 +- messagefoundry/api/app.py | 5 +++-- messagefoundry/pipeline/alerts.py | 11 +++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/PHI.md b/docs/PHI.md index 0711cf8e48..0ae730e32f 100644 --- a/docs/PHI.md +++ b/docs/PHI.md @@ -1821,7 +1821,7 @@ with materially different PHI profiles, so they get their own rows; stream 4 is | **5. `audit_log` table** (SQLite, Postgres, SQL Server) | who / what / **where-from** / when of auth + PHI *access* and admin actions — plus, while `[security].audit_all_authorization_decisions` is on (**default `true`** since BACKLOG #1277, 2026-09-02, which reversed the ADR 0118 §5 `false`; the internal field it desugars to is `audit_all_authz`, whose old `[diagnostics]` TOML spelling is **refused at load** — ADR 0118), an `authz` row for **every** authorization decision including successes. That is the shipped volume of this stream, not an opt-in addition to it: one row per authenticated request on each `require()`-gated route, and this row's retention cell records that **nothing prunes the table** — `actor`, `action`, `channel_id`, `client`, `detail`, `row_hash` | JSON `detail`; **tamper-evident hash chain** over `prev_hash` + the row (the `client` address is **inside** the chained payload — ADR 0150) | the store database | HIPAA §164.312(b) audit controls; incident response; `verify_audit_chain` integrity checks | `GET /audit` requires **`audit:read`**; `GET /audit/export` requires the separate **`audit:export`** and streams CSV with formula-injection neutralisation, recording its own `audit.export` row *before* streaming; `GET /me/security-events` is a per-user view of the same table | **`[retention].audit_days` is reserved and NOT enforced — keep-forever by design** (the audit-retention requirement, ~6 years — **not** chain-breakage; [§8](#8-retention--purge) states the position and cites its source of record) | `detail` is stored **in the clear** (it is not a cipher-covered column): its protection is that writers store at least filter shapes, counts, ids, keyed digests, bounded engine-authored text and client addresses, the `audit_log.detail` row above says which — never bodies or credentials — plus the store ACL and the volume layer. *Corrected 2026-10-16 (vault BACKLOG #3145):* this said writers "only ever store filter shapes, counts and ids" | | **6. `message_events` table** | the per-message disposition timeline — the **complete** vocabulary is `received`, `routed`, `unrouted`, `filtered`, `transformed`, `delivered`, `failed`, `dead`, `error`, `replayed`, `resent`, `reingressed`, `passthrough`, `passthrough_dropped`, `cancelled`, `edit_resend`, `edit_resubmit`, `viewed`, `not_deployed`, the ADR 0154 synchronous-reply pair `reply_returned` / `reply_timeout` (names, counts and `waited_ms` only — **never** a fragment of the partner's reply body), and `reply_dropped`, a delivery whose 2xx reply body was over the byte bound and dropped (the status and the bound only, vault BACKLOG #3040) (CI asserts this list against the engine's own `MESSAGE_EVENT_KINDS`). `[diagnostics].message_events` can thin the set, but never below the compliance floor `viewed` / `dead` / `error` / `failed` / `not_deployed` / `reply_timeout` / `reply_dropped` | rows: `message_id`, `ts`, `event`, `destination`, `detail` | the store database | operator timeline on the message-detail view; the `viewed` row is the HIPAA PHI-access record | `GET /messages/{id}` under **`messages:view_raw`** + `require_phi_read`; the read itself writes a `viewed` event **and** a `message_view` audit row | no dedicated window — `purge_message_bodies` sets `message_events.detail` to `NULL` in the same transaction that blanks the body, so it inherits `[retention].messages_days` | `detail` is `safe_text()`-scrubbed **then** cipher-encrypted (AAD `("message_events","detail",message_id,ts,event)`). Verbosity gate `[diagnostics].message_events` = `all` (default) / `errors` / `off`, with a **compliance floor that can never be thinned**: `viewed`, `dead`, `error`, `failed`, `not_deployed`, `reply_timeout`, `reply_dropped` are retained at every level (`reply_timeout` is the one row that explains a "we called you and got a 504" complaint, so an instance that thinned its logs would lose exactly the record it is later asked for; `reply_dropped` is the one row that answers "you ignored our reply", for the same reason) | | **7. `connection_event` table — DEFAULT ON** (`[diagnostics].connection_events = true`) | transport/lifecycle events per connection: `established`, `closed` (reason `eof`, `idle_timeout` or `frame_deadline` — a fixed vocabulary chosen by the listener, never free text), `idle_timeout`, `at_capacity`, `peer_not_allowlisted`, `frame_oversize`, `framing_error`, `peer_reset`, the MLLP listener's `handler_error` (BACKLOG #1619 -- the inbound handler faulted on a frame the listener read cleanly, such as a store outage at the ingress commit; the reason is scrubbed at the source like every other), the MLLP and HTTP listeners' `tls_handshake_failed` (vault BACKLOG #2613 and #3302 -- an inbound TLS handshake failed, on any event loop; the first at once, then one per listener per minute carrying the count since the last. The reason is a count, an exception class name and OpenSSL's reason mnemonic, never the peer's bytes or certificate. The event names a peer only where the MLLP listener runs its own handshakes, which is the standard asyncio loop; where the loop runs them, which is uvloop and every HTTP listener, the peer is not known, and a handshake that only runs past its bound is not counted), the DATABASE poll source's `row_undecodable` (BACKLOG #1662 — a polled row the source cannot turn into a body; the reason is scrubbed at the source and names the exception type or the operator's own configured column, never a row value), the FILE poll source's quarantine kinds `file_oversize` / `file_decompress_failed` / `file_content_mismatch` / `file_scan_rejected` (BACKLOG #1621 — a drop moved to `.error`; the reason is a size, a content type or a scrubbed codec or scanner message, never the file name), the inbound-HTTP intake-auth refusals `intake_auth_failed` / `auth_subject_denied` / `auth_rate_limited` (ADR 0154 D6 — peer address and mode only; **never** the credential, a prefix of it, or its length. Each of these also writes a tamper-evident audit-log row — the copy that survives an operator turning this diagnostics stream off), plus the runner's `connection_lost` / `connection_restored`. That is the whole vocabulary, asserted in CI against the literal emit call sites in `transports/` and the pipeline runner **and** cross-checked against the console's own filter tuple. The MLLP, raw-TCP, X12 and HTTP listeners emit these; the **DATABASE poll source** emits `row_undecodable` and nothing else (it dials out, so there is no lifecycle to report and its rows carry a **NULL peer column**); the **FILE poll source** emits only its four quarantine kinds, also with a NULL peer column; the **DICOM inbound C-STORE SCP** emits none — the runner injects the sink onto **every** source (`wiring_runner.py`, over the base-class `on_connection_event` field), so that connector *has* the wiring and simply never calls it — so this stream covers those four listeners plus those poll-source events and the runner's outbound-lane transitions — not literally every connection. A DICOM association's connects and refusals are therefore **absent** from this stream. The **`ISA`/`IEA`-framed X12 inbound** was equally absent until BACKLOG #1665 and is not any more: it emits the same seven kinds as the raw-TCP listener | rows: `ts`, `connection`, `transport`, `direction`, `kind`, `peer_host`, `message_id` (correlation hint), `reason` | the store database, **all three backends** | Corepoint-style transport diagnostics — "did the sender connect, and why did it drop" | `GET /events` and `GET /connections/{name}/events` under **`monitoring:read`** (**not** a PHI permission) with per-channel RBAC — an out-of-scope `connection=` is 403'd *and* audited — server-clamped to ≤1000 rows. `reason` alone is gated on `messages:view_summary` and masked as `****` until the audited per-event `reveal=` act (BACKLOG #2443; the `connection_event.reason` row of §3's PL-2 block says how) | `[retention].connection_event_retention_hours` (its own **hours** window); 0 inherits `[retention].messages_days`; both 0 = keep forever. Plain age `DELETE` (metadata-only) | **`reason` is free text that can carry sensitive fragments.** Defended twice — `safe_exc()` at the source, `safe_text(reason, limit=200)` at the store (the bound §3's PL-2 block states) — then cipher-encrypted (AAD `("connection_event","reason",connection,ts,kind)`). Every other column is config metadata; the table is documented **metadata-only** — never a frame, body or HL7 field value. Writes are a pure side observer: a bounded in-memory queue drained by a background task outside any handoff transaction, so a flood can never block a listener or pin a message disposition | -| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100, or an account's group search was refused for the engine's own deadline or cap, so a role or scope change in the directory does not revoke that account's sessions, though a disable or deletion still does, vault BACKLOG #3696; for the breaker, a referral and a probe fault the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker judged; for a probe fault the same total of candidates the pass took up, asked or not, faulted ones included; or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username; a paging refusal is instead keyed on that operator account's username, as `directory-reconciler-paging:`, so each account is its own instance, raised as the account enters that state and again after a restart while it stands, and the payload is that key, the closed-set reason slug `group_search_deadline` or `group_search_cap`, the probed count and a fixed sentence naming the cause, never anything else about the account), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), `sandbox_worker_refused` (a `[sandbox].mode = "subprocess"` worker was refused because the graph it loaded differs from the engine's, most often a config file edited on disk with no reload -- vault BACKLOG #2724; keyed on the inbound's name, raised once per refusal, and the payload is that name and a fixed sentence naming the cause and the remedy, never a message body or the refusal text, which names config names and digests and is in the application log), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` under the same key, so for the same file and the same engine identity, at least once the reload's record no longer holds the refusal, once no hop holds any CRL, and when the reload stops -- vault BACKLOG #2904; it is also raised once per held file outside a CA bundle, at the first pass that judges it with no refusal standing, whether or not an alert is open, so it can resolve nothing, and it then clears one an earlier run under that identity left open -- vault BACKLOG #3543. An alert for a file no hop holds now stays open, and so does one an unpinned cluster node raised, because that node's identity changes on every start; [CONFIGURATION.md](CONFIGURATION.md), *The alert resolves itself*, is the source of record), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_held` and `ad_reconcile_aborted`, the latter only under the breaker's or the referral's source: the probe fault's and a paging refusal's instances have no inverse; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | +| **8. `alert_instance` table — default on wherever an `[alerts]` notifier exists** | resolvable operator alerts: `connection_stopped`, `queue_buildup`, `lane_stuck`, `message_stall`, `saturation`, `connection_error`, `storage_threshold`, `cert_expiry`, `secret_rotation`, `initial_credential_expiring` (an UNCLAIMED admin-issued temporary password nearing the instant the login gate stops accepting it -- ASVS 6.4.5, BACKLOG #1141; keyed on `user:`, and the payload is the ISO deadline plus whole hours remaining, repeated in the reason column, never the password), `integrity_drift`, `update_available`, `backup_failed`, `store_privilege_warning` (the store privilege preflight found the store principal over-granted or could not read it -- BACKLOG #305; the payload is the finding, a count and the preflight's summary line, which names the login, the database and role names, never a password or message content), `leadership_acquired`, `dr_activated`, `log_write_failed` (an application-log sink was rolled after a write failure, or is UNWRITABLE and this process's connections were stopped -- BACKLOG #122, ADR 0162; the payload is the sink LABEL, the stage, a `safe_exc` reason and a count of connections stopped, never the record whose write failed), `log_forward_failed` (the off-box log forwarder is absent, losing records or not sending -- BACKLOG #2612; keyed on `forwarder:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, fixed reason words and a count, never a record, the collector's address or an error text; an unpinned cluster node's `node_id` holds this engine's own host name and process id, so the key carries them), `gcm_invocations` (the per-key AES-GCM invocation bound crossing its 2^31 soft warn — ASVS 11.3.4; its payload carries a one-way `key_id` fingerprint plus counters, never key bytes), `approval_stale_requester` (a dual-control release refused because the requester no longer holds the authority the operation needs -- ASVS 8.3.2, BACKLOG #289; keyed on the approval id, and the payload is that id, the operation key and a closed-set reason slug, never the requester's name or the captured params), `approval_too_early` (a dual-control release refused because it arrived before `[approvals].min_dwell_seconds` -- ASVS 2.4.2, BACKLOG #287; keyed on `approval:`, and the payload is that key, the operation key and a fixed reason string, never the approver's or requester's name or the captured params), `audit_write_failed` (the approval gate could not write one of its audit rows -- vault BACKLOG #2255; keyed on `approval:`, or on `config_reload:inline` when an ungated config reload could not write its `config_reload_attempted` row -- vault BACKLOG #2254; the payload is that key and the lost row's audit action name, never the row's detail, a username or the captured params; the ERROR log line beside it holds the detail on some paths and only the exception on others), `ad_reconcile_aborted` (a directory reconciler pass left signed-in accounts unrevoked and needs an operator: the mass-revoke breaker tripped and nothing was revoked, or one or more probes were answered with an LDAP referral and those accounts were left unjudged while the rest of the pass was applied, BACKLOG #2538, or one or more probes raised a fault the engine could not read and those accounts were left unjudged the same way, BACKLOG #3100, or an account's group search was refused for the engine's own deadline or cap, so a role or scope change in the directory does not revoke that account's sessions, though a disable or deletion still does, vault BACKLOG #3696; for the breaker, a referral and a probe fault the payload is a fixed source label, `directory-reconciler` for the breaker, `directory-reconciler-referral` for a referral and `directory-reconciler-probe-fault` for a probe fault so each is its own instance, the closed-set reason slug `mass_revoke_breaker`, `directory_referral` or `probe_fault`, a count (the accounts the breaker judged; for a probe fault the same total of candidates the pass took up, asked or not, faulted ones included; or for a referral the lookups the pass sent, which leaves out an id-less row -- vault BACKLOG #3170) and the latched operator explanation, never a username; a paging refusal is instead keyed on that operator account's username, as `directory-reconciler-paging:`, so each account is its own instance, raised as the account enters that state and again after a restart while it stands, and the payload is that key, the closed-set reason slug `group_search_deadline` or `group_search_cap`, the same count as a probe fault's and a fixed sentence naming the cause, never anything else about the account), `ad_reconcile_held` (the reconciler is holding accounts whose `userAccountControl` it cannot read, ADR 0195; the payload is the same fixed source label, a closed-set reason slug, the count of undetermined accounts and the latched operator explanation, never a username), `ad_session_revoked` (the reconciler revoked a directory principal's sessions; keyed on that operator account's username, with at least a `directory_absent`, `directory_disabled`, `directory_undetermined`, `directory_object_id_missing`, `roles_changed` or `scope_changed` reason), `approval_approver_provenance` (a dual-control release, or the resolution of an interrupted one, went ahead with an approver or resolver account created, re-passworded or TOTP-enrolled after the request -- BACKLOG #315, #2125; keyed on `approval:`, or `approval::resolve` for a resolution, and the payload is the operation key and the closed-set change slugs, never a username or the captured params), `administrator_granted` (the built-in Administrator role was granted by account creation, a role change, a directory group newly mapped to it -- BACKLOG #315 --, a directory sign-in whose role sync newly gave the account the role -- vault BACKLOG #2610 -- or a directory reconcile pass whose role re-diff did -- vault BACKLOG #3141; keyed on `user:` or `ad-group:`, with the grant route and the granting administrator's username, or `` for a directory sign-in or reconcile pass), `no_enabled_administrator` (a directory sign-in's role sync or the session reconciler took Administrator from its last enabled holder -- vault BACKLOG #3229; keyed on `user:`, and the payload is `via`, `ad-sync` or `ad-reconcile`, and a fixed reason sentence, never message content), `intake_paused` (the engine paused intake because the staged backlog went over `[inbound].max_staged_depth` or the SQLite volume fell below `[retention].min_free_disk_mb` -- BACKLOG #290; keyed on `intake:staged_depth` or `intake:disk_floor`, with `@node:` or `@shard:` appended when several engine processes share the store (BACKLOG #2272), and the payload is the reason, a count or a MiB figure, its limit and the store backend's name, never message content), `config_changed` (a start loaded a config whose fingerprint differs from the store's baseline -- vault BACKLOG #2597; keyed on `config:`, and the payload is the two digests, the node and engine shard labels, and the action, actor username and time of the row the older digest came from, never the config directory, a git commit or message content), `crl_reload_refused` (the CRL reload refused a replaced CRL file, so running TLS contexts keep the copy they hold -- vault BACKLOG #2904; keyed on `crl_reload:`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the file's path, a short class token for the remedy, a count of contexts, the engine's own fixed remedy sentence and, when a start would refuse the file too, a `start_refuses` flag, never the refusal's reason, which quotes CRL issuer names and dates; that reason is in the application log, and in the `detail` of the CRL expiry alert once the held copy nears its `nextUpdate`), `remote_debug_refused` (the engine's remote-debugging guard refused scripts that another process injected into the engine process -- vault BACKLOG #2742; keyed on `remote_debug:refused`, with `@node:` or `@shard:` appended when several engine processes share the store, and the payload is that key, the count of scripts refused since the alert was last raised, the count since the process started and a fixed sentence built from the two, never a script's file name, which the injecting process chooses; that name is in the application log, in the guard's WARNING line), `sandbox_worker_refused` (a `[sandbox].mode = "subprocess"` worker was refused because the graph it loaded differs from the engine's, most often a config file edited on disk with no reload -- vault BACKLOG #2724; keyed on the inbound's name, raised once per refusal, and the payload is that name and a fixed sentence naming the cause and the remedy, never a message body or the refusal text, which names config names and digests and is in the application log), and the six security signals of the audit-stream rule layer -- vault BACKLOG #2613: `signin_failure_burst` (keyed on `signin:`, never the typed username), `access_denied_burst`, `body_view_burst` and `bulk_export` (each keyed on `account:`), `log_level_debug` (keyed on `logging:debug`) and `posture_loosened` (keyed on `posture:start`); the payload is a count and a fixed sentence holding a level name, switch names or a username, never a message body, a message id or an audit row's detail. The reachable **inverse** signals — `connection_restored`, `leadership_lost`, `dr_released`, `store_privilege_clean`, `intake_resumed`, `crl_reload_applied` (it resolves `crl_reload_refused` under the same key, so for the same file and the same engine identity, at least once the reload's record no longer holds the refusal, once no hop holds any CRL, and when the reload stops -- vault BACKLOG #2904; it is also raised once per held file outside a CA bundle, at the first pass that judges it with no refusal standing, whether or not an alert is open, so it can resolve nothing, and it then clears one an earlier run under that identity left open -- vault BACKLOG #3543. An alert for a file no hop holds now stays open, and so does one an unpinned cluster node raised, because that node's identity changes on every start; [CONFIGURATION.md](CONFIGURATION.md), *The alert resolves itself*, is the source of record), `ad_reconcile_breaker_cleared`, `ad_reconcile_hold_released` (the last two resolve `ad_reconcile_held` and `ad_reconcile_aborted`, the latter only under the breaker's or the referral's source: the probe fault's and a paging refusal's instances have no inverse; when they do, and where they never do, is stated once in [CONFIGURATION.md](CONFIGURATION.md#when-the-reconcilers-alerts-resolve-themselves) -- BACKLOG #2136) — are never rows here: `_record_state` routes an inverse through `_AUTO_RESOLVE` to `resolve_alert_instances_for`, never to `upsert_alert_instance`. (One more mapped key, `connection_started`, is emitted by no code path today.) | rows: `event_type`, `connection`, `severity`, `status`, `first_seen`, `last_seen`, `count`, `reason`, `acked_by`, `acked_at`, `resolved_at`, `suspended_until`, `escalation_tier` | the store database, **all three backends** | the operator alert list — acknowledge / resolve / suspend. Durable state is recorded **before** any suppression or throttle return, so a muted alert still leaves a record | `GET /alerts/active` under **`monitoring:diagnose`** (**not** a PHI permission) with the same per-channel scope, and `reason` alone gated on `messages:view_summary` and masked as `****` until the audited per-alert `reveal=` act (BACKLOG #2443; the `alert_instance.reason` row of §3's PL-2 block says how); ack/resolve/suspend/**resume** are POSTs on the same tier, and the separate read-only `GET /alerts/rules` view sits on its own gate | shares the connection-event window; **only RESOLVED instances are DELETEd**, by `resolved_at` — an open or acknowledged condition is never aged out from under an operator | **`reason` is free text** taken from the event's `detail`/`reason`: `safe_text(reason, limit=200)` (the bound §3's PL-2 block states) then cipher-encrypted (AAD `("alert_instance","reason",event_type,connection)` — the de-dup grain, so one AAD covers both the INSERT and the re-fire UPDATE). | | **9. `response` rows with `kind='ack_sent'` — DEFAULT ON** (`[diagnostics].response_sent = true`) | the ACK/NAK the engine returned to an inbound sender, under a sentinel destination `\x1fack:` | rows: `ack_code` (`AA`/`AE`/`AR`/`CA`/`CE`/`CR`), `ack_phase` (`decode`/`parse`/`strict`/`ingest`), `outcome`, `body`, `detail` | the store database | "what did we actually reply, and why" — the operator's answer to a sender disputing an ACK | `GET /messages/{id}/responses` under `messages:read` + `require_phi_read`; the `body` only for a caller who also holds `messages:view_raw` and `messages:view_summary`; every read writes a `response.read` audit row | `body`, `detail` and `resp_headers` are set to `NULL` in place by `purge_message_bodies` on the message-body window, on all three backends | **PHI fail-safe:** the ACK **body** is stored **only when the store cipher is active** — on a keyless store it is `NULL` rather than plaintext — and every NAK passes no body at all, so the offending field value is never persisted. The disposition metadata (`ack_code`/`ack_phase`/`outcome`) is non-PHI and always captured; `detail` is `safe_text`-scrubbed, held to the 200-character bound §3's PL-2 block states, and encrypted | | **10. `[alerts]` webhook transport** (off by default — `webhook_url` unset) | one HTTPS POST per alert, carrying every non-underscore event key as JSON | JSON | the operator's webhook endpoint (Slack/Teams/PagerDuty/custom) | operator notification | **`https` only** — a plaintext `http://` webhook URL is refused at construction unless the `MEFOR_ALLOW_INSECURE_TLS` escape is set (and then a warning is logged); since #329 this path routes that escape through the clamped `weakened_tls_escape_permitted(posture)` (the instance posture threaded from the API lifespan), so on an enforcing-PHI instance the escape is inert and a cleartext webhook POST stays refused — the same clamp as the connectors, no longer the raw escape. Redirects are refused; an optional `webhook_allowed_hosts` egress allowlist gates the host | the endpoint's | **carries the alert's `detail`/`reason` free text** (`safe_exc()`-scrubbed at the emit sites, but **not** re-run through `safe_text` on this path). Internal `_`-prefixed keys (per-rule recipients, rule id, cooldown) are stripped before send, so recipient addresses never cross the wire | | **11. `[alerts]` SMTP transport — operator alert list** (off unless `email_smtp_host` + `email_from` + ≥1 `email_to`) | one email per alert; default subject `[MessageFoundry] — `, default body every non-underscore event key as `k: v` | plain text (always kept — never HTML-only); optional HTML alternative | the operators' mailboxes | operator notification | `smtp_allowed_hosts` egress allowlist; the SMTP password comes from `MEFOR_ALERTS_EMAIL_PASSWORD` or a `[secrets]` provider, never the config file; per-send timeout `email_timeout` | the mail system's | carries the same `detail`/`reason` free text as the webhook. #138 operator templates are constrained to a **closed non-PHI variable allowlist** validated fail-closed at config load. **Transport posture:** `send_plain_email` builds an explicit **verifying** context (chain + hostname + strict RFC 5280, TLS 1.2 floor) via `tls_policy.build_smtp_tls_context()` and passes it to `starttls()`, anchored to the OS roots, `[alerts].email_tls_ca_file`, or `[tls].internal_ca_file` — the same factory the EMAIL and DIRECT *message destinations* use, so all three SMTP cells now share one policy ([#323](BACKLOG.md), closed 2026-08-02). Before that this call passed **no** context and Python's stdlib default applied (`ssl._create_stdlib_context` **is** `ssl._create_unverified_context` — `CERT_NONE`, `check_hostname = False`), leaving the hop encrypted but unauthenticated. There is still **no hop gradient or attestation on this path** — unlike the connectors, this cell is constructed outside the `active_hop_posture` scope, so its deviations (`email_use_tls = false`, or `email_tls_verify = false`) are gated by a `[security].allow_unverified_alert_smtp_tls` **acknowledgment switch at the serve gate** rather than by the clamped escape: on an enforcing PHI instance `serve` refuses to start without it, and permits + `AUDIT`-logs the start with it. Both deviations are named by `security_loosenings()` and reported by `messagefoundry check`'s `alert-smtp-tls` advisory | diff --git a/messagefoundry/api/app.py b/messagefoundry/api/app.py index b51d14837d..00b89f4826 100644 --- a/messagefoundry/api/app.py +++ b/messagefoundry/api/app.py @@ -10395,8 +10395,9 @@ def _alert_reconcile_plan(plan: ReconcilePlan, auth: AuthService, sink: AlertSin point. Their rows stand, except a row that was refused: that account has neither its row nor its alert. ``AuthService._audit_reconciler_row`` says which errors are soft. - Each standing alert but the probe fault's also gets its inverse, which pages nobody and - resolves the open instance (BACKLOG #2136): ``ad_reconcile_breaker_cleared`` and ``ad_reconcile_hold_released``, and + Each standing alert but the probe fault's and a paging refusal's also gets its inverse, which + pages nobody and resolves the open instance (BACKLOG #2136): ``ad_reconcile_breaker_cleared`` + and ``ad_reconcile_hold_released``, and ``ad_reconcile_breaker_cleared`` under the referral's source for the referral's instance. The auth service decides when a pass is evidence of a clear (``plan.breaker_clear``, ``plan.hold_clear``, ``plan.referral_clear``; see ``AuthService._mark_reconcile_clears``), so diff --git a/messagefoundry/pipeline/alerts.py b/messagefoundry/pipeline/alerts.py index 17879bc618..b247ffa40e 100644 --- a/messagefoundry/pipeline/alerts.py +++ b/messagefoundry/pipeline/alerts.py @@ -606,9 +606,11 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s instance and throttle. ``reason`` is one of those five closed-set slugs. ``probed`` is, for the breaker, how many principals the pass judged, its denominator; for a probe fault and a paging refusal, the same total of candidates the pass took up, asked or not, which counts - the accounts it left unjudged and any id-less row; and for a referral how many probes the - pass sent to the directory, which leaves out an id-less row (BACKLOG #3170). ``detail`` is - the operator-facing explanation the auth service latches. No PHI. Emitted by the + any id-less row and, for a probe fault, the faulted accounts it left unjudged; and for a + referral how many probes the pass sent to the directory, which leaves out an id-less row + (BACKLOG #3170). A paging refusal concerns one account, and its count is still the pass's. + ``detail`` is the operator-facing explanation the auth service latches, or for a paging + refusal a fixed sentence naming its cause. No PHI. Emitted by the API-lifespan reconciler task, never from ``auth/``. A whole-directory outage is NOT this event: it is audited as ``auth.ad_reconcile_skipped`` and pages nothing, because the accounts are fine.""" @@ -1179,7 +1181,8 @@ def ad_reconcile_aborted(self, name: str, *, reason: str, probed: int, detail: s # The count is the candidates the pass took up for the breaker, a probe fault and a # paging refusal, or a referral's probes asked (BACKLOG #3170); see # AlertSink.ad_reconcile_aborted. - "ALERT ad_reconcile_aborted: %r left accounts unrevoked, counting %d principal(s) " + "ALERT ad_reconcile_aborted: %r left accounts unrevoked or unjudged, in a pass counting " + "%d principal(s) " "(%s): %s", name, probed,